From bdbb644d8ead8fa5956d564d13631f6f9917d232 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 6 Mar 2022 01:57:46 +0100 Subject: [PATCH] Complete reorganization of process control --- maloja/apis/__init__.py | 30 ++++++++++++-------- maloja/apis/_apikeys.py | 14 ++++++++- maloja/apis/native_v1.py | 53 +++++++++++++++++++++-------------- maloja/database/__init__.py | 3 +- maloja/globalconf.py | 16 +---------- maloja/proccontrol/control.py | 6 +++- maloja/proccontrol/setup.py | 4 ++- maloja/server.py | 4 +-- maloja/upgrade.py | 5 ++-- 9 files changed, 79 insertions(+), 56 deletions(-) diff --git a/maloja/apis/__init__.py b/maloja/apis/__init__.py index 87f17a3..3dfd490 100644 --- a/maloja/apis/__init__.py +++ b/maloja/apis/__init__.py @@ -1,22 +1,28 @@ -from . import native_v1 -from .audioscrobbler import Audioscrobbler -from .audioscrobbler_legacy import AudioscrobblerLegacy -from .listenbrainz import Listenbrainz +from ._apikeys import apikeystore + import copy from bottle import redirect, request, response from urllib.parse import urlencode -native_apis = [ - native_v1.api -] -standardized_apis = [ - Listenbrainz(), - Audioscrobbler(), - AudioscrobblerLegacy() -] + def init_apis(server): + + from . import native_v1 + from .audioscrobbler import Audioscrobbler + from .audioscrobbler_legacy import AudioscrobblerLegacy + from .listenbrainz import Listenbrainz + + native_apis = [ + native_v1.api + ] + standardized_apis = [ + Listenbrainz(), + Audioscrobbler(), + AudioscrobblerLegacy() + ] + for api in native_apis: api.mount(server=server,path="apis/"+api.__apipath__) diff --git a/maloja/apis/_apikeys.py b/maloja/apis/_apikeys.py index 67096ab..3d19670 100644 --- a/maloja/apis/_apikeys.py +++ b/maloja/apis/_apikeys.py @@ -1,4 +1,16 @@ -from ..globalconf import apikeystore +### API KEYS +### symmetric keys are fine since we hopefully use HTTPS + +from doreah.keystore import KeyStore +from doreah.logging import log + +from ..globalconf import data_dir + +apikeystore = KeyStore(file=data_dir['clients']("apikeys.yml"),save_endpoint="/apis/mlj_1/apikeys") +from .. import upgrade +upgrade.upgrade_apikeys() + +log("Authenticated Machines: " + ", ".join([k for k in apikeystore]),module='apis') # skip regular authentication if api key is present in request # an api key now ONLY permits scrobbling tracks, no other admin tasks diff --git a/maloja/apis/native_v1.py b/maloja/apis/native_v1.py index 2f1029d..ed9485a 100644 --- a/maloja/apis/native_v1.py +++ b/maloja/apis/native_v1.py @@ -1,9 +1,14 @@ -from ..database import * -from ..globalconf import malojaconfig, apikeystore + +from .. import database +from ..globalconf import malojaconfig + + + from ..__pkginfo__ import VERSION from ..malojauri import uri_to_internal from .. import utilities from ._apikeys import api_key_correct, checkAPIkey +from . import apikeystore from bottle import response, static_file @@ -12,6 +17,12 @@ from nimrodel import EAPI as API from nimrodel import Multi + +from doreah.auth import authenticated_api, authenticated_api_with_alternate + + + + api = API(delay=True) api.__apipath__ = "mlj_1" @@ -48,7 +59,7 @@ def server_info(): "name":malojaconfig["NAME"], "version":VERSION.split("."), "versionstring":VERSION, - "db_status":dbstatus + "db_status":database.dbstatus } @@ -60,7 +71,7 @@ def get_scrobbles_external(**keys): k_filter, k_time, _, k_amount, _ = uri_to_internal(keys,api=True) ckeys = {**k_filter, **k_time, **k_amount} - result = get_scrobbles(**ckeys) + result = database.get_scrobbles(**ckeys) offset = (k_amount.get('page') * k_amount.get('perpage')) if k_amount.get('perpage') is not math.inf else 0 result = result[offset:] @@ -85,7 +96,7 @@ def get_scrobbles_num_external(**keys): k_filter, k_time, _, k_amount, _ = uri_to_internal(keys) ckeys = {**k_filter, **k_time, **k_amount} - result = get_scrobbles_num(**ckeys) + result = database.get_scrobbles_num(**ckeys) return {"amount":result} @@ -95,14 +106,14 @@ def get_tracks_external(**keys): k_filter, _, _, _, _ = uri_to_internal(keys,forceArtist=True) ckeys = {**k_filter} - result = get_tracks(**ckeys) + result = database.get_tracks(**ckeys) return {"list":result} @api.get("artists") def get_artists_external(): - result = get_artists() + result = database.get_artists() return {"list":result} @@ -114,7 +125,7 @@ def get_charts_artists_external(**keys): _, k_time, _, _, _ = uri_to_internal(keys) ckeys = {**k_time} - result = get_charts_artists(**ckeys) + result = database.get_charts_artists(**ckeys) return {"list":result} @@ -124,7 +135,7 @@ def get_charts_tracks_external(**keys): k_filter, k_time, _, _, _ = uri_to_internal(keys,forceArtist=True) ckeys = {**k_filter, **k_time} - result = get_charts_tracks(**ckeys) + result = database.get_charts_tracks(**ckeys) return {"list":result} @@ -135,7 +146,7 @@ def get_pulse_external(**keys): k_filter, k_time, k_internal, k_amount, _ = uri_to_internal(keys) ckeys = {**k_filter, **k_time, **k_internal, **k_amount} - results = get_pulse(**ckeys) + results = database.get_pulse(**ckeys) return {"list":results} @@ -146,7 +157,7 @@ def get_performance_external(**keys): k_filter, k_time, k_internal, k_amount, _ = uri_to_internal(keys) ckeys = {**k_filter, **k_time, **k_internal, **k_amount} - results = get_performance(**ckeys) + results = database.get_performance(**ckeys) return {"list":results} @@ -157,7 +168,7 @@ def get_top_artists_external(**keys): _, k_time, k_internal, _, _ = uri_to_internal(keys) ckeys = {**k_time, **k_internal} - results = get_top_artists(**ckeys) + results = database.get_top_artists(**ckeys) return {"list":results} @@ -170,7 +181,7 @@ def get_top_tracks_external(**keys): # IMPLEMENT THIS FOR TOP TRACKS OF ARTIST AS WELL? - results = get_top_tracks(**ckeys) + results = database.get_top_tracks(**ckeys) return {"list":results} @@ -181,7 +192,7 @@ def artist_info_external(**keys): k_filter, _, _, _, _ = uri_to_internal(keys,forceArtist=True) ckeys = {**k_filter} - return artist_info(**ckeys) + return database.artist_info(**ckeys) @@ -194,12 +205,12 @@ def track_info_external(artist:Multi[str],**keys): k_filter, _, _, _, _ = uri_to_internal(keys,forceTrack=True) ckeys = {**k_filter} - return track_info(**ckeys) + return database.track_info(**ckeys) @api.get("compare") def compare_external(**keys): - return compare(keys["remote"]) + return database.compare(keys["remote"]) @@ -214,7 +225,7 @@ def get_post_scrobble(artist:Multi,**keys): time = keys.get("time") if time is not None: time = int(time) - return incoming_scrobble(artists,title,album=album,duration=duration,time=time) + return database.incoming_scrobble(artists,title,album=album,duration=duration,time=time) @api.post("newscrobble") @authenticated_api_with_alternate(api_key_correct) @@ -237,7 +248,7 @@ def post_scrobble(artist:Multi=None,**keys): keys['fix'] = keys.get("nofix") is None if keys.get('time') is not None: keys['time'] = int(keys.get('time')) - return incoming_scrobble(**keys,client=request.malojaclient) + return database.incoming_scrobble(**keys,client=request.malojaclient) # TODO: malojaclient needs to be converted to proper argument in doreah @@ -264,15 +275,15 @@ def import_rulemodule(**keys): @authenticated_api def rebuild(**keys): log("Database rebuild initiated!") - sync() + database.sync() dbstatus['rebuildinprogress'] = True from ..proccontrol.tasks.fixexisting import fix fix() global cla, coa cla = CleanerAgent() coa = CollectorAgent() - build_db() - invalidate_caches() + database.build_db() + database.invalidate_caches() diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index 52ca949..8c19dbd 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -7,7 +7,8 @@ from .. import utilities from ..malojatime import register_scrobbletime, time_stamps, ranges, alltime from ..malojauri import uri_to_internal, internal_to_uri, compose_querystring from ..thirdparty import proxy_scrobble_all -from ..globalconf import data_dir, malojaconfig, apikeystore +from ..globalconf import data_dir, malojaconfig +from ..apis import apikeystore #db from . import sqldb from . import cached diff --git a/maloja/globalconf.py b/maloja/globalconf.py index e0107cb..70373c7 100644 --- a/maloja/globalconf.py +++ b/maloja/globalconf.py @@ -1,7 +1,7 @@ import os from doreah.configuration import Configuration from doreah.configuration import types as tp -from doreah.keystore import KeyStore + from .__pkginfo__ import VERSION @@ -311,20 +311,6 @@ config( -### API KEYS -### symmetric keys are fine since we hopefully use HTTPS - - -apikeystore = KeyStore(file=data_dir['clients']("apikeys.yml"),save_endpoint="/apis/mlj_1/apikeys") -from . import upgrade -upgrade.upgrade_apikeys() - -print("Authenticated Machines: " + ", ".join([k for k in apikeystore])) - - - - - diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py index d68f4b3..6d9209c 100644 --- a/maloja/proccontrol/control.py +++ b/maloja/proccontrol/control.py @@ -79,7 +79,10 @@ def stop(): print("Maloja stopped!") return True - +def onlysetup(): + print_header_info() + setup() + print("Setup complete!") def direct(): print_header_info() @@ -111,6 +114,7 @@ def main(*args,**kwargs): "stop":stop, "run":direct, "debug":debug, + "setup":onlysetup, "import":tasks.loadlastfm, "backup":tasks.backuphere, # "update":update, diff --git a/maloja/proccontrol/setup.py b/maloja/proccontrol/setup.py index 37bdc1d..5066818 100644 --- a/maloja/proccontrol/setup.py +++ b/maloja/proccontrol/setup.py @@ -4,7 +4,8 @@ from doreah.io import col, ask, prompt from doreah import auth import os -from ..globalconf import data_dir, dir_settings, malojaconfig, apikeystore +from ..globalconf import data_dir, dir_settings, malojaconfig + # EXTERNAL API KEYS @@ -47,6 +48,7 @@ def setup(): # OWN API KEY + from ..apis import apikeystore if len(apikeystore) == 0: answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) if answer: diff --git a/maloja/server.py b/maloja/server.py index 8318bf8..cd82beb 100644 --- a/maloja/server.py +++ b/maloja/server.py @@ -21,9 +21,9 @@ from . import database from .database.jinjaview import JinjaDBConnection from .utilities import get_track_image, get_artist_image from .malojauri import uri_to_internal, remove_identical -from .globalconf import malojaconfig, apikeystore, data_dir +from .globalconf import malojaconfig, data_dir from .jinjaenv.context import jinja_environment -from .apis import init_apis +from .apis import init_apis, apikeystore from .proccontrol.profiler import profile diff --git a/maloja/upgrade.py b/maloja/upgrade.py index 7126131..df7c487 100644 --- a/maloja/upgrade.py +++ b/maloja/upgrade.py @@ -6,7 +6,8 @@ import re from doreah.logging import log from doreah.io import col -from .globalconf import data_dir, dir_settings, apikeystore +from .globalconf import data_dir, dir_settings +from . import apis def upgrade_apikeys(): @@ -17,7 +18,7 @@ def upgrade_apikeys(): from doreah import tsv clients = tsv.parse(oldfile,"string","string") for key,identifier in clients: - apikeystore[identifier] = key + apis.apikeystore[identifier] = key os.remove(oldfile) except: pass