From b17060184b6897b18cf8af28a3817c9989aac96f Mon Sep 17 00:00:00 2001 From: Krateng Date: Tue, 21 Apr 2020 18:11:16 +0200 Subject: [PATCH] Fixing DB now creates individual patch files --- maloja/__pkginfo__.py | 2 +- maloja/fixexisting.py | 66 +++++++++++++------------ maloja/lastfmconverter.py | 82 ++++++++++++++++---------------- scrobblers/maloja-scrobbler.zip | Bin 29156 -> 29167 bytes 4 files changed, 73 insertions(+), 77 deletions(-) diff --git a/maloja/__pkginfo__.py b/maloja/__pkginfo__.py index 8ab3ccd..191560d 100644 --- a/maloja/__pkginfo__.py +++ b/maloja/__pkginfo__.py @@ -5,7 +5,7 @@ author = { "email":"maloja@krateng.dev", "github": "krateng" } -version = 2,3,6 +version = 2,3,7 versionstr = ".".join(str(n) for n in version) links = { "pypi":"malojaserver", diff --git a/maloja/fixexisting.py b/maloja/fixexisting.py index 2dba2b6..9ee87c4 100644 --- a/maloja/fixexisting.py +++ b/maloja/fixexisting.py @@ -22,43 +22,41 @@ def fix(): datestr = now.strftime("%Y/%m/%d") timestr = now.strftime("%H:%M:%S") - with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile: + patchfolder = datadir("logs","dbfix",nowstr) + os.makedirs(patchfolder) - logfile.write("Database fix initiated on " + datestr + " " + timestr + " UTC") - logfile.write("\n\n") - - for filename in os.listdir(datadir("scrobbles")): - if filename.endswith(".tsv"): - filename_new = filename + "_new" - - with open(datadir("scrobbles",filename_new),"w") as newfile: - with open(datadir("scrobbles",filename),"r") as oldfile: - - for l in oldfile: - - a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l) - r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6\7",l) - - a = a.replace("␟",";") - - (al,t) = wendigo.fullclean(a,t) - a = "␟".join(al) - newfile.write(r1 + a + r2 + t + r3 + "\n") + #with open(datadir("logs","dbfix",nowstr + ".log"),"a") as logfile: - #os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn) - with open(datadir("scrobbles",filename_new),"r") as newfile: - with open(datadir("scrobbles",filename),"r") as oldfile: + for filename in os.listdir(datadir("scrobbles")): + if filename.endswith(".tsv"): + filename_new = filename + "_new" - diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="") - diff = list(diff)[2:] - #log("Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff),module="fixer") - output = "Diff for scrobbles/" + filename + "".join("\n\t" + d for d in diff) - print(output) - logfile.write(output) - logfile.write("\n") + with open(datadir("scrobbles",filename_new),"w") as newfile: + with open(datadir("scrobbles",filename),"r") as oldfile: - os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename)) + for l in oldfile: - with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: - checkfile.write(wendigo.checksums) + a,t = re.sub(exp,r"\3",l), re.sub(exp,r"\5",l) + r1,r2,r3 = re.sub(exp,r"\1\2",l),re.sub(exp,r"\4",l),re.sub(exp,r"\6\7",l) + + a = a.replace("␟",";") + + (al,t) = wendigo.fullclean(a,t) + a = "␟".join(al) + newfile.write(r1 + a + r2 + t + r3 + "\n") + + + #os.system("diff " + "scrobbles/" + fn + "_new" + " " + "scrobbles/" + fn) + with open(datadir("scrobbles",filename_new),"r") as newfile, open(datadir("scrobbles",filename),"r") as oldfile: + + diff = difflib.unified_diff(oldfile.read().split("\n"),newfile.read().split("\n"),lineterm="") + diff = list(diff) + + with open(os.path.join(patchfolder,filename + ".diff"),"w") as patchfile: + patchfile.write("\n".join(diff)) + + os.rename(datadir("scrobbles",filename_new),datadir("scrobbles",filename)) + + with open(datadir("scrobbles",filename + ".rulestate"),"w") as checkfile: + checkfile.write(wendigo.checksums) diff --git a/maloja/lastfmconverter.py b/maloja/lastfmconverter.py index 9f58990..6a70d80 100644 --- a/maloja/lastfmconverter.py +++ b/maloja/lastfmconverter.py @@ -11,59 +11,57 @@ c = CleanerAgent() def convert(input,output): - log = open(input,"r") - outputlog = open(output,"w") - checksumfile = open(output + ".rulestate","w") #this file stores an identifier for all rules that were in place when the corresponding file was created - - stamps = [99999999999999] - - for l in log: - l = l.replace("\n","") - data = l.split(",") - - artist = data[0] - album = data[1] - title = data[2] - time = data[3] + with open(input,"r",encoding="utf-8") as log: + with open(output,"w") as outputlog: - (artists,title) = c.fullclean(artist,title) + stamps = [99999999999999] - artistsstr = "␟".join(artists) + for l in log: + l = l.replace("\n","") + data = l.split(",") + + artist = data[0] + album = data[1] + title = data[2] + time = data[3] - timeparts = time.split(" ") - (h,m) = timeparts[3].split(":") + (artists,title) = c.fullclean(artist,title) - months = {"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9,"Oct":10,"Nov":11,"Dec":12} - - timestamp = int(datetime.datetime(int(timeparts[2]),months[timeparts[1]],int(timeparts[0]),int(h),int(m)).timestamp()) + artistsstr = "␟".join(artists) - ## We prevent double timestamps in the database creation, so we technically don't need them in the files - ## however since the conversion from lastfm to maloja is a one-time thing, we should take any effort to make the file as good as possible - if (timestamp < stamps[-1]): - pass - elif (timestamp == stamps[-1]): - timestamp -= 1 - else: - while(timestamp in stamps): - timestamp -= 1 + timeparts = time.split(" ") + (h,m) = timeparts[3].split(":") - if (timestamp < stamps[-1]): - stamps.append(timestamp) - else: - stamps.insert(0,timestamp) + months = {"Jan":1,"Feb":2,"Mar":3,"Apr":4,"May":5,"Jun":6,"Jul":7,"Aug":8,"Sep":9,"Oct":10,"Nov":11,"Dec":12} + + timestamp = int(datetime.datetime(int(timeparts[2]),months[timeparts[1]],int(timeparts[0]),int(h),int(m)).timestamp()) - entry = "\t".join([str(timestamp),artistsstr,title,album]) - entry = entry.replace("#",r"\num") + ## We prevent double timestamps in the database creation, so we technically don't need them in the files + ## however since the conversion from lastfm to maloja is a one-time thing, we should take any effort to make the file as good as possible + if (timestamp < stamps[-1]): + pass + elif (timestamp == stamps[-1]): + timestamp -= 1 + else: + while(timestamp in stamps): + timestamp -= 1 - outputlog.write(entry) - outputlog.write("\n") + if (timestamp < stamps[-1]): + stamps.append(timestamp) + else: + stamps.insert(0,timestamp) - checksumfile.write(c.checksums) - log.close() - outputlog.close() - checksumfile.close() + entry = "\t".join([str(timestamp),artistsstr,title,album]) + entry = entry.replace("#",r"\num") + + outputlog.write(entry) + outputlog.write("\n") + + with open(output + ".rulestate","w") as checksumfile: + #this file stores an identifier for all rules that were in place when the corresponding file was created + checksumfile.write(c.checksums) diff --git a/scrobblers/maloja-scrobbler.zip b/scrobblers/maloja-scrobbler.zip index c09068600d44e597f03c985763d577d4a574c4b8..6db390db8b7e8d20256c903c3a22d1f56ff0dae7 100644 GIT binary patch delta 1499 zcmaFznDPB%M!o=VW)=}<1_lm>SsP3PHu8CLv49wxqq)>1A*@#A<+>1-LEtkk4$)i|I~3wS>A04%BeM%7pq)&tA2{_kb*j| zS&zm=q4K6be99;Out?3=^nbzLw>&|Ae_xnZX<`tO(!}|8YTik|`e)AOHQmNr{F>9g zN!dvHbF1HdP~^n5pn9{2`l5)n*ZE(EoZ4b^=uE_?HFH`S#$&TshgR+hP?ix0v5n57lgY z8RB+-_0==w=6j-7Xx#XJPQ6Xf!cC4vO6hgAj=zb;S9j-Wb7rrkp9Bk!-S@_>-dZ@Hc@FfbTS z{*WeK-+JME-XQ}Xwhz_wnZ625IH8oAzGR=fsB2zU#5EJgZ8zs{kH39jM^BOOoB3aB zm5=;k?6ZBkuG^!}aM5(1cQOx`GvAUi+iF<*BS%`k@#lh|f0sU|d#Ky;YD9;Je!U?X zxPZM;vMO@wobHM#C!-&$cpO>HnrnPIYJ%#Q?()Yi2l(r)+DtUES@x&yUL78yz^U`+ z`L8)FPLFaw`&Hc04rc4Qb@RDT;5_-a`Yb|4zn0Go+9&U&n|?~&DNQ%gCH408=p|pD zJr|hrc<%n=TIJtP%c|7QO?ken+r!W1X^hytHJbX%BD>bFjzy{uv`1_h8910#b31A|dP&b$j( z85@F~%k>$rF~ax(-i%E4%v``+zyQt#leu$EHDHq16(h3D z_?BAJfAsJ}^33;qbA**nCTjojDCSID{OJoY#Cm{M3Zqz=U!VdvrQw=>2`Ev(Vg)(; zP;ws76af@dt`sP0Fx=Tv0FDJT8rc+Zz{(a%F}|K`Q78}hD?W?CTIvfWn71@4Po7mM ds|!oHZx2sF4P{m~P@>=iLL*iNhUGaR9soWdCEEZ1 delta 1571 zcmaF=nDNPDM!o=VW)=}<1_lm>zoilW8~HrBSd?40_-&5nQj>(RT9udULRbcY&$u{1 z%DD2?yC%!UsMRmEP)X5PR`T-B%DYYm>F$M7PEFcry?e@p|8aBIUMN}Xu;BN*moGba zuQ-d#9I+MB`f4wfUUfjG??Ct@{`(KtwF8r%fd*sGz z&I?B;npjTryK=wKW$t{1?%;3l6?ZIFT~{xEi$h-LTVG3nlQpyM?vwR(4$9kAoL@6K z-pJ<3Vb-6saN$!I89}SvBI?f;My-_#TX`ij)n#Vq6Uja6-M@TE3fVZF!}phOdm#^d z@XAL@i=IAKDc$J(J1%72NtG?(N52MY?(F~At#eJtw(X7L#eRdTb+Nl07ILmP_g`kj zba|zW)AS_H`}aPSEnFq=-+I0xpWb7IW`)yloZO#wR_FiAxUq5j=M9^Vt+<(Lwyb_x zK=@Z?MCgJ8wIKExBP1xJ;$JeefP!+ed$IzUu{k^09V)pkg@X|yb|_Vmkr$-Q>Q=VJ zO$G)Ax5*#U#Or%6p3OUCz{CEbdOp+Fpb67lZ=Lhl!``K2ZmOZ&J2Cg$$8XuQ<;o-) zvmgC=wBOcg5&s9F8Ncils}Hs~cK$Uz=+D3BW`0z5`;SVS78a%7dY7lq)K9jXy>dZu z?q!qja|K1&0%k0{zUoQsp~-%6y_Z*T{*P$fg^_emvnnmb}ni| z*Zp00w~9(eEM8%gz5Z!y{Ts1wh2n>M9hINf9}6`T(n}2sUuL|xg!_30=6O_AD1fB#$NRoS=8rlNe?oEwKX^=|vgiW*?obRT7NF)}c`Vq#zr0Q$Z- zvm~`xzjCuchAbmXLib$%&4!sx%(9>Wx|d=6ONontVY&hX122OLP%#K7G%T5XAxBQ5 z;Yq-f!mg`~4bvX^rtAaKP=0_nBa=Nd7cl)ZFo0rcvRJOEM#H|Gd?kLMB8J85!HS`L zh+;@)0V^&>R@@l+=Hd>ZCKSa;CSS-k(Sb?gGZ~)04Dv+Wmo)OkFfimM=4Ga(7MJK{ z73b%1F(`ms%fQGW!2t6H+&mO-A-S|W&m7^c#VrT%dh1mlQW)S=%m6MhCcEbwAVPQl zkrlBRp^GS8CeP28U|ceJN4^q53oO`hxtys0$>lJ`C}E6beqw42zbX2APXa V2Y9oxff5xT5E`*EFwDsT@c>sTR_p)(