diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 2fd02e5..f1bf847 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -33,5 +33,5 @@ resources = [ ] commands = { - "maloja":"controller:main" + "maloja":"proccontrol.control:main" } diff --git a/maloja/controller.py b/maloja/controller.py deleted file mode 100755 index fa61563..0000000 --- a/maloja/controller.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import sys -import signal -import os -import shutil -from distutils import dir_util -import stat -import pathlib -import pkg_resources -from doreah.control import mainfunction -from doreah.io import col, ask, prompt - -from .globalconf import datadir -from .backup import backup - - - - -def copy_initial_local_files(): - folder = pkg_resources.resource_filename(__name__,"data_files") - #shutil.copy(folder,DATA_DIR) - dir_util.copy_tree(folder,datadir(),update=False) - - -def setup(): - - copy_initial_local_files() - - from doreah import settings - - # EXTERNAL API KEYS - apikeys = { - "LASTFM_API_KEY":"Last.fm API Key", - "FANARTTV_API_KEY":"Fanart.tv API Key", - "SPOTIFY_API_ID":"Spotify Client ID", - "SPOTIFY_API_SECRET":"Spotify Client Secret" - } - - SKIP = settings.get_settings("SKIP_SETUP") - - print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") - for k in apikeys: - key = settings.get_settings(k) - if key is None: - print("\t" + "Currently not using a " + apikeys[k] + " for image display.") - elif key == "ASK": - print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") - key = prompt("",types=(str,),default=None,skip=SKIP) - settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) - else: - print("\t" + apikeys[k] + " found.") - - - # OWN API KEY - if os.path.exists(datadir("clients/authenticated_machines.tsv")): - pass - else: - answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) - if answer: - import random - key = "" - for i in range(64): - key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) - print("Your API Key: " + col["yellow"](key)) - with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: - keyfile.write(key + "\t" + "Default Generated Key") - else: - pass - - - if settings.get_settings("NAME") is None: - name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP) - settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) - - if settings.get_settings("SEND_STATS") is None: - answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP) - if answer: - settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) - else: - settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) - - -def getInstance(): - try: - output = subprocess.check_output(["pidof","Maloja"]) - pid = int(output) - return pid - except: - return None - -def getInstanceSupervisor(): - try: - output = subprocess.check_output(["pidof","maloja_supervisor"]) - pid = int(output) - return pid - except: - return None - -def start(): - setup() - try: - #p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - print(col["green"]("Maloja started!")) - - from doreah import settings - port = settings.get_settings("WEB_PORT") - - print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") - print("If you're installing this on your local machine, these links should get you there:") - print("\t" + col["blue"]("http://localhost:" + str(port))) - print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup")) - return True - except: - print("Error while starting Maloja.") - return False - -def restart(): - wasrunning = stop() - start() - return wasrunning - -def stop(): - pid_sv = getInstanceSupervisor() - if pid_sv is not None: - os.kill(pid_sv,signal.SIGTERM) -# return True - -# else: -# print("Server is not running") -# return False - - - pid = getInstance() - if pid is not None: -# print("Server is not running") -# return False -# pass -# else: - os.kill(pid,signal.SIGTERM) -# print("Maloja stopped! PID: " + str(pid)) - if pid is not None or pid_sv is not None: - return True - else: - return False - - -def loadlastfm(filename): - - if not os.path.exists(filename): - print("File could not be found.") - return - - if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): - print("Already imported Last.FM data. Overwrite? [y/N]") - if input().lower() in ["y","yes","yea","1","positive","true"]: - pass - else: - return - print("Please wait...") - from .lastfmconverter import convert - convert(filename,datadir("scrobbles/lastfmimport.tsv")) - #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) - print("Successfully imported your Last.FM scrobbles!") - -def direct(): - setup() - from . import server - -def backuphere(): - backup(folder=os.getcwd()) - -def update(): - os.system("pip3 install malojaserver --upgrade --no-cache-dir") - restart() - -def fixdb(): - from .fixexisting import fix - fix() - -@mainfunction({"l":"level"},shield=True) -def main(action,*args,**kwargs): - actions = { - "start":restart, - "restart":restart, - "stop":stop, - "import":loadlastfm, - "debug":direct, - "backup":backuphere, - "update":update, - "fix":fixdb, - "run":direct - } - if action in actions: actions[action](*args,**kwargs) - else: print("Valid commands: " + " ".join(a for a in actions)) - - return True - -#if __name__ == "__main__": -# main() diff --git a/maloja/data_files/settings/default.ini b/maloja/data_files/settings/default.ini index 37d2ab4..e6415d7 100644 --- a/maloja/data_files/settings/default.ini +++ b/maloja/data_files/settings/default.ini @@ -59,7 +59,7 @@ SCROBBLES_GOLD = 250 SCROBBLES_PLATINUM = 500 SCROBBLES_DIAMOND = 1000 # name for comparisons -NAME = "A Maloja User" +NAME = None [Misc] @@ -67,7 +67,5 @@ EXPERIMENTAL_FEATURES = no USE_PYHP = no #not recommended at the moment USE_JINJA = no #overwrites pyhp preference FEDERATION = yes #does nothing yet -UPDATE_AFTER_CRASH = no #update when server is automatically restarted -DAILY_RESTART = 2 # hour of day. no / none means no daily restarts SKIP_SETUP = no LOGGING = true diff --git a/maloja/database.py b/maloja/database.py index 84aada3..ac94d77 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -854,7 +854,7 @@ def rebuild(**keys): global db_rulestate db_rulestate = False sync() - from .fixexisting import fix + from .proccontrol.tasks.fixexisting import fix fix() global cla, coa cla = CleanerAgent() diff --git a/maloja/proccontrol/control.py b/maloja/proccontrol/control.py new file mode 100644 index 0000000..d0cf53e --- /dev/null +++ b/maloja/proccontrol/control.py @@ -0,0 +1,94 @@ +import subprocess +from doreah import settings +from doreah.control import mainfunction +from doreah.io import col +import os +import signal + +from .setup import setup +from . import tasks + +def getInstance(): + try: + output = subprocess.check_output(["pidof","Maloja"]) + pid = int(output) + return pid + except: + return None + +def getInstanceSupervisor(): + try: + output = subprocess.check_output(["pidof","maloja_supervisor"]) + pid = int(output) + return pid + except: + return None + +def restart(): + stop() + start() + +def start(): + if getInstanceSupervisor() is not None: + print("Maloja is already running.") + else: + setup() + try: + #p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + sp = subprocess.Popen(["python3","-m","maloja.proccontrol.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + print(col["green"]("Maloja started!")) + + port = settings.get_settings("WEB_PORT") + + print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.") + print("If you're installing this on your local machine, these links should get you there:") + print("\t" + col["blue"]("http://localhost:" + str(port))) + print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup")) + return True + except: + print("Error while starting Maloja.") + return False + + +def stop(): + + pid_sv = getInstanceSupervisor() + if pid_sv is not None: + os.kill(pid_sv,signal.SIGTERM) + + pid = getInstance() + if pid is not None: + os.kill(pid,signal.SIGTERM) + + if pid is not None or pid_sv is not None: + print("Maloja stopped!") + return True + else: + return False + + + +def direct(): + setup() + from .. import server + + + +@mainfunction({"l":"level"},shield=True) +def main(action,*args,**kwargs): + actions = { + "start":start, + "restart":restart, + "stop":stop, + "run":direct, + "debug":direct, + + "import":tasks.loadlastfm, + "backup":tasks.backuphere, + # "update":update, + "fix":tasks.fixdb + } + if action in actions: actions[action](*args,**kwargs) + else: print("Valid commands: " + " ".join(a for a in actions)) + + return True diff --git a/maloja/proccontrol/setup.py b/maloja/proccontrol/setup.py new file mode 100644 index 0000000..ad6752a --- /dev/null +++ b/maloja/proccontrol/setup.py @@ -0,0 +1,71 @@ +import pkg_resources +from distutils import dir_util +from doreah import settings +from doreah.io import col, ask, prompt +import os + +from ..globalconf import datadir + + +# EXTERNAL API KEYS +apikeys = { + "LASTFM_API_KEY":"Last.fm API Key", + "FANARTTV_API_KEY":"Fanart.tv API Key", + "SPOTIFY_API_ID":"Spotify Client ID", + "SPOTIFY_API_SECRET":"Spotify Client Secret" +} + + + +def copy_initial_local_files(): + folder = pkg_resources.resource_filename("maloja","data_files") + #shutil.copy(folder,DATA_DIR) + dir_util.copy_tree(folder,datadir(),update=False) + + + +def setup(): + + copy_initial_local_files() + SKIP = settings.get_settings("SKIP_SETUP") + + print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.") + for k in apikeys: + key = settings.get_settings(k) + if key is None: + print("\t" + "Currently not using a " + apikeys[k] + " for image display.") + elif key == "ASK": + print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.") + key = prompt("",types=(str,),default=None,skip=SKIP) + settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True) + else: + print("\t" + apikeys[k] + " found.") + + + # OWN API KEY + if os.path.exists(datadir("clients/authenticated_machines.tsv")): + pass + else: + answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP) + if answer: + import random + key = "" + for i in range(64): + key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))) + print("Your API Key: " + col["yellow"](key)) + with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile: + keyfile.write(key + "\t" + "Default Generated Key") + else: + pass + + + if settings.get_settings("NAME") is None: + name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP) + settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True) + + if settings.get_settings("SEND_STATS") is None: + answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP) + if answer: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True) + else: + settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True) diff --git a/maloja/proccontrol/supervisor.py b/maloja/proccontrol/supervisor.py new file mode 100644 index 0000000..9fa05b0 --- /dev/null +++ b/maloja/proccontrol/supervisor.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +import os + +import subprocess +import setproctitle +import signal +from doreah.logging import log +from doreah.settings import get_settings + +from .control import getInstance + + +setproctitle.setproctitle("maloja_supervisor") + +def update(): + log("Updating...",module="supervisor") + try: + os.system("pip3 install maloja --upgrade --no-cache-dir") + except: + log("Could not update.",module="supervisor") + +def start(): + try: + p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) + return p + except e: + log("Error starting Maloja: " + str(e),module="supervisor") + + + +while True: + log("Maloja is not running, starting...",module="supervisor") + if get_settings("UPDATE_AFTER_CRASH"): + update() + process = start() + + process.wait() diff --git a/maloja/proccontrol/tasks/__init__.py b/maloja/proccontrol/tasks/__init__.py new file mode 100644 index 0000000..daaf86f --- /dev/null +++ b/maloja/proccontrol/tasks/__init__.py @@ -0,0 +1,34 @@ +import os +from .lastfmconverter import convert +from .backup import backup +from .fixexisting import fix + +from ...globalconf import datadir +from ..control import restart +from doreah.io import ask + +def loadlastfm(filename): + + if not os.path.exists(filename): + print("File could not be found.") + return + + if os.path.exists(datadir("scrobbles/lastfmimport.tsv")): + overwrite = ask("Already imported Last.FM data. Overwrite?",default=False) + if not overwrite: return + print("Please wait...") + + convert(filename,datadir("scrobbles/lastfmimport.tsv")) + #os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv")) + print("Successfully imported your Last.FM scrobbles!") + + +def backuphere(): + backup(folder=os.getcwd()) + +def update(): + os.system("pip3 install malojaserver --upgrade --no-cache-dir") + restart() + +def fixdb(): + fix() diff --git a/maloja/backup.py b/maloja/proccontrol/tasks/backup.py similarity index 74% rename from maloja/backup.py rename to maloja/proccontrol/tasks/backup.py index 3a7e9da..d3aefce 100644 --- a/maloja/backup.py +++ b/maloja/proccontrol/tasks/backup.py @@ -2,7 +2,10 @@ import tarfile from datetime import datetime import glob import os -from .globalconf import datadir +from ...globalconf import datadir +from pathlib import PurePath + +from doreah.logging import log user_files = { @@ -25,6 +28,8 @@ def backup(folder,level="full"): for g in selected_files: real_files += glob.glob(datadir(g)) + log("Creating backup of " + str(len(real_files)) + " files...") + now = datetime.utcnow() timestr = now.strftime("%Y_%m_%d_%H_%M_%S") filename = "maloja_backup_" + timestr + ".tar.gz" @@ -32,4 +37,7 @@ def backup(folder,level="full"): assert not os.path.exists(archivefile) with tarfile.open(name=archivefile,mode="x:gz") as archive: for f in real_files: - archive.add(f) + p = PurePath(f) + r = p.relative_to(datadir()) + archive.add(f,arcname=r) + log("Backup created!") diff --git a/maloja/fixexisting.py b/maloja/proccontrol/tasks/fixexisting.py similarity index 92% rename from maloja/fixexisting.py rename to maloja/proccontrol/tasks/fixexisting.py index b0c39a3..76036c4 100644 --- a/maloja/fixexisting.py +++ b/maloja/proccontrol/tasks/fixexisting.py @@ -1,7 +1,7 @@ import os -from .globalconf import datadir +from ...globalconf import datadir import re -from .cleanup import CleanerAgent +from ...cleanup import CleanerAgent from doreah.logging import log import difflib import datetime @@ -35,9 +35,10 @@ def fix(): #with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile: - + log("Fixing database...") for filename in os.listdir(datadir("scrobbles")): if filename.endswith(".tsv"): + log("Fix file " + filename) filename_new = filename + "_new" with open(datadir("scrobbles",filename_new),"w") as newfile: @@ -68,3 +69,5 @@ def fix(): with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: checkfile.write(wendigo.checksums) + + log("Database fixed!") diff --git a/maloja/lastfmconverter.py b/maloja/proccontrol/tasks/lastfmconverter.py similarity index 96% rename from maloja/lastfmconverter.py rename to maloja/proccontrol/tasks/lastfmconverter.py index 6a70d80..bb5a6dc 100644 --- a/maloja/lastfmconverter.py +++ b/maloja/proccontrol/tasks/lastfmconverter.py @@ -1,6 +1,6 @@ import os, datetime, re -from .cleanup import * -from .utilities import * +from ...cleanup import * +from ...utilities import * diff --git a/maloja/supervisor.py b/maloja/supervisor.py deleted file mode 100644 index 2846252..0000000 --- a/maloja/supervisor.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -import os - -import subprocess -import time -import setproctitle -import signal -from datetime import datetime -from doreah.logging import log -from doreah.settings import get_settings - - -setproctitle.setproctitle("maloja_supervisor") - -lastrestart = () - -def get_pid(): - try: - output = subprocess.check_output(["pidof","Maloja"]) - return int(output) - except: - return None - -def update(): - log("Updating...",module="supervisor") - try: - os.system("pip3 install maloja --upgrade --no-cache-dir") - except: - log("Could not update.",module="supervisor") - -def start(): - try: - p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) - - except e: - log("Error starting Maloja: " + str(e),module="supervisor") - - - -while True: - - - now = datetime.now() - today = now.year, now.month, now.day - - pid = get_pid() - - if pid: - - restart = get_settings("DAILY_RESTART") - if restart not in [None,False]: - if today != lastrestart: - if now.hour == restart: - log("Daily restart...",module="supervisor") - os.kill(pid,signal.SIGTERM) - start() - lastrestart = today - - else: - log("Maloja is not running, starting...",module="supervisor") - if get_settings("UPDATE_AFTER_CRASH"): - update() - start() - lastrestart = today - - - time.sleep(60)