mirror of
https://github.com/krateng/maloja.git
synced 2025-04-23 03:50:28 +03:00
Merge branch 'processcontrolrework'
This commit is contained in:
Krateng
commit
895fcfd8c3
@ -33,5 +33,5 @@ resources = [
|
||||
]
|
||||
|
||||
commands = {
|
||||
"maloja":"controller:main"
|
||||
"maloja":"proccontrol.control:main"
|
||||
}
|
||||
|
||||
@ -1,202 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import signal
|
||||
import os
|
||||
import shutil
|
||||
from distutils import dir_util
|
||||
import stat
|
||||
import pathlib
|
||||
import pkg_resources
|
||||
from doreah.control import mainfunction
|
||||
from doreah.io import col, ask, prompt
|
||||
|
||||
from .globalconf import datadir
|
||||
from .backup import backup
|
||||
|
||||
|
||||
|
||||
|
||||
def copy_initial_local_files():
|
||||
folder = pkg_resources.resource_filename(__name__,"data_files")
|
||||
#shutil.copy(folder,DATA_DIR)
|
||||
dir_util.copy_tree(folder,datadir(),update=False)
|
||||
|
||||
|
||||
def setup():
|
||||
|
||||
copy_initial_local_files()
|
||||
|
||||
from doreah import settings
|
||||
|
||||
# EXTERNAL API KEYS
|
||||
apikeys = {
|
||||
"LASTFM_API_KEY":"Last.fm API Key",
|
||||
"FANARTTV_API_KEY":"Fanart.tv API Key",
|
||||
"SPOTIFY_API_ID":"Spotify Client ID",
|
||||
"SPOTIFY_API_SECRET":"Spotify Client Secret"
|
||||
}
|
||||
|
||||
SKIP = settings.get_settings("SKIP_SETUP")
|
||||
|
||||
print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.")
|
||||
for k in apikeys:
|
||||
key = settings.get_settings(k)
|
||||
if key is None:
|
||||
print("\t" + "Currently not using a " + apikeys[k] + " for image display.")
|
||||
elif key == "ASK":
|
||||
print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.")
|
||||
key = prompt("",types=(str,),default=None,skip=SKIP)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True)
|
||||
else:
|
||||
print("\t" + apikeys[k] + " found.")
|
||||
|
||||
|
||||
# OWN API KEY
|
||||
if os.path.exists(datadir("clients/authenticated_machines.tsv")):
|
||||
pass
|
||||
else:
|
||||
answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
|
||||
if answer:
|
||||
import random
|
||||
key = ""
|
||||
for i in range(64):
|
||||
key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")))
|
||||
print("Your API Key: " + col["yellow"](key))
|
||||
with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile:
|
||||
keyfile.write(key + "\t" + "Default Generated Key")
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
if settings.get_settings("NAME") is None:
|
||||
name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True)
|
||||
|
||||
if settings.get_settings("SEND_STATS") is None:
|
||||
answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP)
|
||||
if answer:
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True)
|
||||
else:
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True)
|
||||
|
||||
|
||||
def getInstance():
|
||||
try:
|
||||
output = subprocess.check_output(["pidof","Maloja"])
|
||||
pid = int(output)
|
||||
return pid
|
||||
except:
|
||||
return None
|
||||
|
||||
def getInstanceSupervisor():
|
||||
try:
|
||||
output = subprocess.check_output(["pidof","maloja_supervisor"])
|
||||
pid = int(output)
|
||||
return pid
|
||||
except:
|
||||
return None
|
||||
|
||||
def start():
|
||||
setup()
|
||||
try:
|
||||
#p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
sp = subprocess.Popen(["python3","-m","maloja.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
print(col["green"]("Maloja started!"))
|
||||
|
||||
from doreah import settings
|
||||
port = settings.get_settings("WEB_PORT")
|
||||
|
||||
print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.")
|
||||
print("If you're installing this on your local machine, these links should get you there:")
|
||||
print("\t" + col["blue"]("http://localhost:" + str(port)))
|
||||
print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup"))
|
||||
return True
|
||||
except:
|
||||
print("Error while starting Maloja.")
|
||||
return False
|
||||
|
||||
def restart():
|
||||
wasrunning = stop()
|
||||
start()
|
||||
return wasrunning
|
||||
|
||||
def stop():
|
||||
pid_sv = getInstanceSupervisor()
|
||||
if pid_sv is not None:
|
||||
os.kill(pid_sv,signal.SIGTERM)
|
||||
# return True
|
||||
|
||||
# else:
|
||||
# print("Server is not running")
|
||||
# return False
|
||||
|
||||
|
||||
pid = getInstance()
|
||||
if pid is not None:
|
||||
# print("Server is not running")
|
||||
# return False
|
||||
# pass
|
||||
# else:
|
||||
os.kill(pid,signal.SIGTERM)
|
||||
# print("Maloja stopped! PID: " + str(pid))
|
||||
if pid is not None or pid_sv is not None:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def loadlastfm(filename):
|
||||
|
||||
if not os.path.exists(filename):
|
||||
print("File could not be found.")
|
||||
return
|
||||
|
||||
if os.path.exists(datadir("scrobbles/lastfmimport.tsv")):
|
||||
print("Already imported Last.FM data. Overwrite? [y/N]")
|
||||
if input().lower() in ["y","yes","yea","1","positive","true"]:
|
||||
pass
|
||||
else:
|
||||
return
|
||||
print("Please wait...")
|
||||
from .lastfmconverter import convert
|
||||
convert(filename,datadir("scrobbles/lastfmimport.tsv"))
|
||||
#os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv"))
|
||||
print("Successfully imported your Last.FM scrobbles!")
|
||||
|
||||
def direct():
|
||||
setup()
|
||||
from . import server
|
||||
|
||||
def backuphere():
|
||||
backup(folder=os.getcwd())
|
||||
|
||||
def update():
|
||||
os.system("pip3 install malojaserver --upgrade --no-cache-dir")
|
||||
restart()
|
||||
|
||||
def fixdb():
|
||||
from .fixexisting import fix
|
||||
fix()
|
||||
|
||||
@mainfunction({"l":"level"},shield=True)
|
||||
def main(action,*args,**kwargs):
|
||||
actions = {
|
||||
"start":restart,
|
||||
"restart":restart,
|
||||
"stop":stop,
|
||||
"import":loadlastfm,
|
||||
"debug":direct,
|
||||
"backup":backuphere,
|
||||
"update":update,
|
||||
"fix":fixdb,
|
||||
"run":direct
|
||||
}
|
||||
if action in actions: actions[action](*args,**kwargs)
|
||||
else: print("Valid commands: " + " ".join(a for a in actions))
|
||||
|
||||
return True
|
||||
|
||||
#if __name__ == "__main__":
|
||||
# main()
|
||||
@ -59,7 +59,7 @@ SCROBBLES_GOLD = 250
|
||||
SCROBBLES_PLATINUM = 500
|
||||
SCROBBLES_DIAMOND = 1000
|
||||
# name for comparisons
|
||||
NAME = "A Maloja User"
|
||||
NAME = None
|
||||
|
||||
[Misc]
|
||||
|
||||
@ -67,7 +67,5 @@ EXPERIMENTAL_FEATURES = no
|
||||
USE_PYHP = no #not recommended at the moment
|
||||
USE_JINJA = no #overwrites pyhp preference
|
||||
FEDERATION = yes #does nothing yet
|
||||
UPDATE_AFTER_CRASH = no #update when server is automatically restarted
|
||||
DAILY_RESTART = 2 # hour of day. no / none means no daily restarts
|
||||
SKIP_SETUP = no
|
||||
LOGGING = true
|
||||
|
||||
@ -854,7 +854,7 @@ def rebuild(**keys):
|
||||
global db_rulestate
|
||||
db_rulestate = False
|
||||
sync()
|
||||
from .fixexisting import fix
|
||||
from .proccontrol.tasks.fixexisting import fix
|
||||
fix()
|
||||
global cla, coa
|
||||
cla = CleanerAgent()
|
||||
|
||||
94
maloja/proccontrol/control.py
Normal file
94
maloja/proccontrol/control.py
Normal file
@ -0,0 +1,94 @@
|
||||
import subprocess
|
||||
from doreah import settings
|
||||
from doreah.control import mainfunction
|
||||
from doreah.io import col
|
||||
import os
|
||||
import signal
|
||||
|
||||
from .setup import setup
|
||||
from . import tasks
|
||||
|
||||
def getInstance():
|
||||
try:
|
||||
output = subprocess.check_output(["pidof","Maloja"])
|
||||
pid = int(output)
|
||||
return pid
|
||||
except:
|
||||
return None
|
||||
|
||||
def getInstanceSupervisor():
|
||||
try:
|
||||
output = subprocess.check_output(["pidof","maloja_supervisor"])
|
||||
pid = int(output)
|
||||
return pid
|
||||
except:
|
||||
return None
|
||||
|
||||
def restart():
|
||||
stop()
|
||||
start()
|
||||
|
||||
def start():
|
||||
if getInstanceSupervisor() is not None:
|
||||
print("Maloja is already running.")
|
||||
else:
|
||||
setup()
|
||||
try:
|
||||
#p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
sp = subprocess.Popen(["python3","-m","maloja.proccontrol.supervisor"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
print(col["green"]("Maloja started!"))
|
||||
|
||||
port = settings.get_settings("WEB_PORT")
|
||||
|
||||
print("Visit your server address (Port " + str(port) + ") to see your web interface. Visit /setup to get started.")
|
||||
print("If you're installing this on your local machine, these links should get you there:")
|
||||
print("\t" + col["blue"]("http://localhost:" + str(port)))
|
||||
print("\t" + col["blue"]("http://localhost:" + str(port) + "/setup"))
|
||||
return True
|
||||
except:
|
||||
print("Error while starting Maloja.")
|
||||
return False
|
||||
|
||||
|
||||
def stop():
|
||||
|
||||
pid_sv = getInstanceSupervisor()
|
||||
if pid_sv is not None:
|
||||
os.kill(pid_sv,signal.SIGTERM)
|
||||
|
||||
pid = getInstance()
|
||||
if pid is not None:
|
||||
os.kill(pid,signal.SIGTERM)
|
||||
|
||||
if pid is not None or pid_sv is not None:
|
||||
print("Maloja stopped!")
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
|
||||
def direct():
|
||||
setup()
|
||||
from .. import server
|
||||
|
||||
|
||||
|
||||
@mainfunction({"l":"level"},shield=True)
|
||||
def main(action,*args,**kwargs):
|
||||
actions = {
|
||||
"start":start,
|
||||
"restart":restart,
|
||||
"stop":stop,
|
||||
"run":direct,
|
||||
"debug":direct,
|
||||
|
||||
"import":tasks.loadlastfm,
|
||||
"backup":tasks.backuphere,
|
||||
# "update":update,
|
||||
"fix":tasks.fixdb
|
||||
}
|
||||
if action in actions: actions[action](*args,**kwargs)
|
||||
else: print("Valid commands: " + " ".join(a for a in actions))
|
||||
|
||||
return True
|
||||
71
maloja/proccontrol/setup.py
Normal file
71
maloja/proccontrol/setup.py
Normal file
@ -0,0 +1,71 @@
|
||||
import pkg_resources
|
||||
from distutils import dir_util
|
||||
from doreah import settings
|
||||
from doreah.io import col, ask, prompt
|
||||
import os
|
||||
|
||||
from ..globalconf import datadir
|
||||
|
||||
|
||||
# EXTERNAL API KEYS
|
||||
apikeys = {
|
||||
"LASTFM_API_KEY":"Last.fm API Key",
|
||||
"FANARTTV_API_KEY":"Fanart.tv API Key",
|
||||
"SPOTIFY_API_ID":"Spotify Client ID",
|
||||
"SPOTIFY_API_SECRET":"Spotify Client Secret"
|
||||
}
|
||||
|
||||
|
||||
|
||||
def copy_initial_local_files():
|
||||
folder = pkg_resources.resource_filename("maloja","data_files")
|
||||
#shutil.copy(folder,DATA_DIR)
|
||||
dir_util.copy_tree(folder,datadir(),update=False)
|
||||
|
||||
|
||||
|
||||
def setup():
|
||||
|
||||
copy_initial_local_files()
|
||||
SKIP = settings.get_settings("SKIP_SETUP")
|
||||
|
||||
print("Various external services can be used to display images. If not enough of them are set up, only local images will be used.")
|
||||
for k in apikeys:
|
||||
key = settings.get_settings(k)
|
||||
if key is None:
|
||||
print("\t" + "Currently not using a " + apikeys[k] + " for image display.")
|
||||
elif key == "ASK":
|
||||
print("\t" + "Please enter your " + apikeys[k] + ". If you do not want to use one at this moment, simply leave this empty and press Enter.")
|
||||
key = prompt("",types=(str,),default=None,skip=SKIP)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{k:key},create_new=True)
|
||||
else:
|
||||
print("\t" + apikeys[k] + " found.")
|
||||
|
||||
|
||||
# OWN API KEY
|
||||
if os.path.exists(datadir("clients/authenticated_machines.tsv")):
|
||||
pass
|
||||
else:
|
||||
answer = ask("Do you want to set up a key to enable scrobbling? Your scrobble extension needs that key so that only you can scrobble tracks to your database.",default=True,skip=SKIP)
|
||||
if answer:
|
||||
import random
|
||||
key = ""
|
||||
for i in range(64):
|
||||
key += str(random.choice(list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ")))
|
||||
print("Your API Key: " + col["yellow"](key))
|
||||
with open(datadir("clients/authenticated_machines.tsv"),"w") as keyfile:
|
||||
keyfile.write(key + "\t" + "Default Generated Key")
|
||||
else:
|
||||
pass
|
||||
|
||||
|
||||
if settings.get_settings("NAME") is None:
|
||||
name = prompt("Please enter your name. This will be displayed e.g. when comparing your charts to another user. Leave this empty if you would not like to specify a name right now.",default="Generic Maloja User",skip=SKIP)
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"NAME":name},create_new=True)
|
||||
|
||||
if settings.get_settings("SEND_STATS") is None:
|
||||
answer = ask("I would like to know how many people use Maloja. Would it be okay to send a daily ping to my server (this contains no data that isn't accessible via your web interface already)?",default=True,skip=SKIP)
|
||||
if answer:
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":True,"PUBLIC_URL":None},create_new=True)
|
||||
else:
|
||||
settings.update_settings(datadir("settings/settings.ini"),{"SEND_STATS":False},create_new=True)
|
||||
37
maloja/proccontrol/supervisor.py
Normal file
37
maloja/proccontrol/supervisor.py
Normal file
@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
|
||||
import subprocess
|
||||
import setproctitle
|
||||
import signal
|
||||
from doreah.logging import log
|
||||
from doreah.settings import get_settings
|
||||
|
||||
from .control import getInstance
|
||||
|
||||
|
||||
setproctitle.setproctitle("maloja_supervisor")
|
||||
|
||||
def update():
|
||||
log("Updating...",module="supervisor")
|
||||
try:
|
||||
os.system("pip3 install maloja --upgrade --no-cache-dir")
|
||||
except:
|
||||
log("Could not update.",module="supervisor")
|
||||
|
||||
def start():
|
||||
try:
|
||||
p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
return p
|
||||
except e:
|
||||
log("Error starting Maloja: " + str(e),module="supervisor")
|
||||
|
||||
|
||||
|
||||
while True:
|
||||
log("Maloja is not running, starting...",module="supervisor")
|
||||
if get_settings("UPDATE_AFTER_CRASH"):
|
||||
update()
|
||||
process = start()
|
||||
|
||||
process.wait()
|
||||
34
maloja/proccontrol/tasks/__init__.py
Normal file
34
maloja/proccontrol/tasks/__init__.py
Normal file
@ -0,0 +1,34 @@
|
||||
import os
|
||||
from .lastfmconverter import convert
|
||||
from .backup import backup
|
||||
from .fixexisting import fix
|
||||
|
||||
from ...globalconf import datadir
|
||||
from ..control import restart
|
||||
from doreah.io import ask
|
||||
|
||||
def loadlastfm(filename):
|
||||
|
||||
if not os.path.exists(filename):
|
||||
print("File could not be found.")
|
||||
return
|
||||
|
||||
if os.path.exists(datadir("scrobbles/lastfmimport.tsv")):
|
||||
overwrite = ask("Already imported Last.FM data. Overwrite?",default=False)
|
||||
if not overwrite: return
|
||||
print("Please wait...")
|
||||
|
||||
convert(filename,datadir("scrobbles/lastfmimport.tsv"))
|
||||
#os.system("python3 -m maloja.lastfmconverter " + filename + " " + datadir("scrobbles/lastfmimport.tsv"))
|
||||
print("Successfully imported your Last.FM scrobbles!")
|
||||
|
||||
|
||||
def backuphere():
|
||||
backup(folder=os.getcwd())
|
||||
|
||||
def update():
|
||||
os.system("pip3 install malojaserver --upgrade --no-cache-dir")
|
||||
restart()
|
||||
|
||||
def fixdb():
|
||||
fix()
|
||||
@ -2,7 +2,10 @@ import tarfile
|
||||
from datetime import datetime
|
||||
import glob
|
||||
import os
|
||||
from .globalconf import datadir
|
||||
from ...globalconf import datadir
|
||||
from pathlib import PurePath
|
||||
|
||||
from doreah.logging import log
|
||||
|
||||
|
||||
user_files = {
|
||||
@ -25,6 +28,8 @@ def backup(folder,level="full"):
|
||||
for g in selected_files:
|
||||
real_files += glob.glob(datadir(g))
|
||||
|
||||
log("Creating backup of " + str(len(real_files)) + " files...")
|
||||
|
||||
now = datetime.utcnow()
|
||||
timestr = now.strftime("%Y_%m_%d_%H_%M_%S")
|
||||
filename = "maloja_backup_" + timestr + ".tar.gz"
|
||||
@ -32,4 +37,7 @@ def backup(folder,level="full"):
|
||||
assert not os.path.exists(archivefile)
|
||||
with tarfile.open(name=archivefile,mode="x:gz") as archive:
|
||||
for f in real_files:
|
||||
archive.add(f)
|
||||
p = PurePath(f)
|
||||
r = p.relative_to(datadir())
|
||||
archive.add(f,arcname=r)
|
||||
log("Backup created!")
|
||||
@ -1,7 +1,7 @@
|
||||
import os
|
||||
from .globalconf import datadir
|
||||
from ...globalconf import datadir
|
||||
import re
|
||||
from .cleanup import CleanerAgent
|
||||
from ...cleanup import CleanerAgent
|
||||
from doreah.logging import log
|
||||
import difflib
|
||||
import datetime
|
||||
@ -35,9 +35,10 @@ def fix():
|
||||
|
||||
#with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile:
|
||||
|
||||
|
||||
log("Fixing database...")
|
||||
for filename in os.listdir(datadir("scrobbles")):
|
||||
if filename.endswith(".tsv"):
|
||||
log("Fix file " + filename)
|
||||
filename_new = filename + "_new"
|
||||
|
||||
with open(datadir("scrobbles",filename_new),"w") as newfile:
|
||||
@ -68,3 +69,5 @@ def fix():
|
||||
|
||||
with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile:
|
||||
checkfile.write(wendigo.checksums)
|
||||
|
||||
log("Database fixed!")
|
||||
@ -1,6 +1,6 @@
|
||||
import os, datetime, re
|
||||
from .cleanup import *
|
||||
from .utilities import *
|
||||
from ...cleanup import *
|
||||
from ...utilities import *
|
||||
|
||||
|
||||
|
||||
@ -1,67 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
import setproctitle
|
||||
import signal
|
||||
from datetime import datetime
|
||||
from doreah.logging import log
|
||||
from doreah.settings import get_settings
|
||||
|
||||
|
||||
setproctitle.setproctitle("maloja_supervisor")
|
||||
|
||||
lastrestart = ()
|
||||
|
||||
def get_pid():
|
||||
try:
|
||||
output = subprocess.check_output(["pidof","Maloja"])
|
||||
return int(output)
|
||||
except:
|
||||
return None
|
||||
|
||||
def update():
|
||||
log("Updating...",module="supervisor")
|
||||
try:
|
||||
os.system("pip3 install maloja --upgrade --no-cache-dir")
|
||||
except:
|
||||
log("Could not update.",module="supervisor")
|
||||
|
||||
def start():
|
||||
try:
|
||||
p = subprocess.Popen(["python3","-m","maloja.server"],stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
|
||||
except e:
|
||||
log("Error starting Maloja: " + str(e),module="supervisor")
|
||||
|
||||
|
||||
|
||||
while True:
|
||||
|
||||
|
||||
now = datetime.now()
|
||||
today = now.year, now.month, now.day
|
||||
|
||||
pid = get_pid()
|
||||
|
||||
if pid:
|
||||
|
||||
restart = get_settings("DAILY_RESTART")
|
||||
if restart not in [None,False]:
|
||||
if today != lastrestart:
|
||||
if now.hour == restart:
|
||||
log("Daily restart...",module="supervisor")
|
||||
os.kill(pid,signal.SIGTERM)
|
||||
start()
|
||||
lastrestart = today
|
||||
|
||||
else:
|
||||
log("Maloja is not running, starting...",module="supervisor")
|
||||
if get_settings("UPDATE_AFTER_CRASH"):
|
||||
update()
|
||||
start()
|
||||
lastrestart = today
|
||||
|
||||
|
||||
time.sleep(60)
|
||||
Loading…
x
Reference in New Issue
Block a user