diff --git a/maloja/cleanup.py b/maloja/cleanup.py index 6145fdb..6580c36 100644 --- a/maloja/cleanup.py +++ b/maloja/cleanup.py @@ -24,19 +24,6 @@ class CleanerAgent: self.rules_artistintitle = {b.lower():c for [a,b,c,d] in raw if a=="artistintitle"} #self.rules_regexartist = [[b,c] for [a,b,c,d] in raw if a=="regexartist"] #self.rules_regextitle = [[b,c] for [a,b,c,d] in raw if a=="regextitle"] - # TODO - - #self.plugin_artistparsers = [] - #self.plugin_titleparsers = [] - #if settings.get_settings("USE_PARSE_PLUGINS"): - # for ep in pkg_resources.iter_entry_points(group='maloja.artistparsers'): - # self.plugin_artistparsers.append(ep.load()) - # for ep in pkg_resources.iter_entry_points(group='maloja.titleparsers'): - # self.plugin_titleparsers.append(ep.load()) - - - # we always need to be able to tell if our current database is made with the current rules - self.checksums = utilities.checksumTSV(datadir("rules")) def fullclean(self,artist,title): diff --git a/maloja/database.py b/maloja/database.py index c214340..6608572 100644 --- a/maloja/database.py +++ b/maloja/database.py @@ -78,8 +78,6 @@ clients = [] lastsync = 0 -# rulestate that the entire current database was built with, or False if the database was built from inconsistent scrobble files -db_rulestate = False try: with open(datadir("known_servers.yml"),"r") as f: @@ -272,15 +270,11 @@ def test_server(key=None): response.status = 403 return "Wrong API key" - elif db_rulestate: - response.status = 204 - return else: - response.status = 205 + response.status = 200 return - # 204 Database server is up and operational - # 205 Database server is up, but DB is not fully built or is inconsistent + # 200 Database server is up and operational # 403 Database server is up, but provided API key is not valid @dbserver.get("serverinfo") @@ -795,8 +789,6 @@ def sapi(path:Multi,**keys): def newrule(**keys): tsv.add_entry(datadir("rules/webmade.tsv"),[k for k in keys]) #addEntry("rules/webmade.tsv",[k for k in keys]) - global db_rulestate - db_rulestate = False def issues(): @@ -960,8 +952,6 @@ def import_rulemodule(**keys): @authenticated_api def rebuild(**keys): log("Database rebuild initiated!") - global db_rulestate - db_rulestate = False sync() from .proccontrol.tasks.fixexisting import fix fix() @@ -1091,8 +1081,6 @@ def build_db(): utilities.update_weekly() utilities.send_stats() - global db_rulestate - db_rulestate = utilities.consistentRulestate(datadir("scrobbles"),cla.checksums) global ISSUES ISSUES = check_issues() @@ -1135,7 +1123,6 @@ def sync(): for e in entries: tsv.add_entries(datadir("scrobbles/" + e + ".tsv"),entries[e],comments=False) #addEntries("scrobbles/" + e + ".tsv",entries[e],escape=False) - utilities.combineChecksums(datadir("scrobbles/" + e + ".tsv"),cla.checksums) #log("Written files",module="debug") diff --git a/maloja/proccontrol/tasks/fixexisting.py b/maloja/proccontrol/tasks/fixexisting.py index 76036c4..c659239 100644 --- a/maloja/proccontrol/tasks/fixexisting.py +++ b/maloja/proccontrol/tasks/fixexisting.py @@ -67,7 +67,5 @@ def fix(): os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename)) - with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: - checkfile.write(wendigo.checksums) log("Database fixed!") diff --git a/maloja/proccontrol/tasks/lastfmconverter.py b/maloja/proccontrol/tasks/lastfmconverter.py index bb5a6dc..4c0cc6e 100644 --- a/maloja/proccontrol/tasks/lastfmconverter.py +++ b/maloja/proccontrol/tasks/lastfmconverter.py @@ -61,7 +61,3 @@ def convert(input,output): outputlog.write(entry) outputlog.write("\n") - - with open(output + ".rulestate","w") as checksumfile: - #this file stores an identifier for all rules that were in place when the corresponding file was created - checksumfile.write(c.checksums) diff --git a/maloja/utilities.py b/maloja/utilities.py index 8c7e4ed..bd56759 100644 --- a/maloja/utilities.py +++ b/maloja/utilities.py @@ -48,85 +48,6 @@ def serialize(obj): - - -##### -## RULESTATE VALIDATION -##### - - - - - -def checksumTSV(folder): - - sums = "" - - for f in os.listdir(folder + "/"): - if (f.endswith(".tsv")): - f = open(folder + "/" + f,"rb") - sums += hashlib.md5(f.read()).hexdigest() + "\n" - f.close() - - return sums - -# returns whether checksums match and sets the checksum to invalid if they don't (or sets the new one if no previous one exists) -def combineChecksums(filename,checksums): - import os - - if os.path.exists(filename + ".rulestate"): - f = open(filename + ".rulestate","r") - oldchecksums = f.read() - f.close() - if oldchecksums == checksums: - # the new checksum given by the calling db server represents the rule state that all current unsaved scrobbles were created under - # if this is the same as the existing one, we're all good - return True - elif (oldchecksums != "INVALID"): - #if not, the file is not consistent to any single rule state (some scrobbles were created with an old ruleset, some not) - f = open(filename + ".rulestate","w") - f.write("INVALID") # this will never match any sha256sum - f.close() - return False - else: - #if the file already says invalid, no need to open it and rewrite - return False - else: - f = open(filename + ".rulestate","w") - f.write(checksums) - f.close() - return True - -# checks ALL files for their rule state. if they are all the same as the current loaded one, the entire database can be assumed to be consistent with the current ruleset -# in any other case, get out -def consistentRulestate(folder,checksums): - - result = [] - for scrobblefile in os.listdir(folder + "/"): - - if (scrobblefile.endswith(".tsv")): - - try: - with open(folder + "/" + scrobblefile + ".rulestate","r") as f: - if f.read() != checksums: - return False - except: - return False - - return True - - - - - - - - - - - - - ##### ## IMAGES #####