From eccf74f3438246d7915651ea08212826528cadc2 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Sun, 19 Feb 2023 01:39:45 -0500 Subject: [PATCH 1/5] Clarify Docker env var requirements This gave me a few minutes' trouble getting Maloja up and running! I'm still not sure if these are exactly all that's required, but I'm pretty sure they _are_ required when running Maloja in Docker. (Possibly related to #147 ?) --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9d54108..7658652 100644 --- a/README.md +++ b/README.md @@ -84,9 +84,13 @@ Pull the [latest image](https://hub.docker.com/r/krateng/maloja) or check out th Of note are these settings which should be passed as environmental variables to the container: +#### First run +* `MALOJA_SKIP_SETUP` -- Make the server setup process non-interactive. Maloja will not work properly without this variable set. +* `MALOJA_FORCE_PASSWORD` -- Set an admin password for Maloja. You will need this to log in on first run. + +#### Always * `MALOJA_DATA_DIRECTORY` -- Set the directory in the container where configuration folders/files should be located * Mount a [volume](https://docs.docker.com/engine/reference/builder/#volume) to the specified directory to access these files outside the container (and to make them persistent) -* `MALOJA_FORCE_PASSWORD` -- Set an admin password for maloja You must publish a port on your host machine to bind to the container's web port (default 42010). The container uses IPv4 per default. From 39a42e915c3f598ece2a4a70c7ccd256be7531ba Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 25 Jun 2023 18:00:23 +0200 Subject: [PATCH 2/5] Initial support for new Spotify export format, GH-215 --- maloja/proccontrol/tasks/import_scrobbles.py | 67 ++++++++++++++++++-- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index b5bf620..4efd0ce 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -37,13 +37,17 @@ def import_scrobbles(inputf): typeid,typedesc = "lastfm","Last.fm" importfunc = parse_lastfm + elif re.match("Streaming_History_Audio.+\.json",filename): + typeid,typedesc = "spotify","Spotify" + importfunc = parse_spotify_lite + elif re.match("endsong_[0-9]+\.json",filename): typeid,typedesc = "spotify","Spotify" - importfunc = parse_spotify_full + importfunc = parse_spotify elif re.match("StreamingHistory[0-9]+\.json",filename): typeid,typedesc = "spotify","Spotify" - importfunc = parse_spotify_lite + importfunc = parse_spotify_lite_legacy elif re.match("maloja_export_[0-9]+\.json",filename): typeid,typedesc = "maloja","Maloja" @@ -81,6 +85,7 @@ def import_scrobbles(inputf): # extra info extrainfo = {} if scrobble.get('album_name'): extrainfo['album_name'] = scrobble['album_name'] + if scrobble.get('album_artist'): extrainfo['album_artist'] = scrobble['album_artist'] # saving this in the scrobble instead of the track because for now it's not meant # to be authorative information, just payload of the scrobble @@ -121,7 +126,7 @@ def import_scrobbles(inputf): return result -def parse_spotify_lite(inputf): +def parse_spotify_lite_legacy(inputf): pth = os.path inputfolder = pth.relpath(pth.dirname(pth.abspath(inputf))) filenames = re.compile(r'StreamingHistory[0-9]+\.json') @@ -171,7 +176,59 @@ def parse_spotify_lite(inputf): print() -def parse_spotify_full(inputf): +def parse_spotify_lite(inputf): + pth = os.path + inputfolder = pth.relpath(pth.dirname(pth.abspath(inputf))) + filenames = re.compile(r'Streaming_History_Audio.+\.json') + inputfiles = [os.path.join(inputfolder,f) for f in os.listdir(inputfolder) if filenames.match(f)] + + if len(inputfiles) == 0: + print("No files found!") + return + + if inputfiles != [inputf]: + print("Spotify files should all be imported together to identify duplicates across the whole dataset.") + if not ask("Import " + ", ".join(col['yellow'](i) for i in inputfiles) + "?",default=True): + inputfiles = [inputf] + + for inputf in inputfiles: + + print("Importing",col['yellow'](inputf),"...") + with open(inputf,'r') as inputfd: + data = json.load(inputfd) + + for entry in data: + + try: + played = int(entry['ms_played'] / 1000) + timestamp = int( + datetime.datetime.strptime(entry['ts'],"%Y-%m-%dT%H:%M:%SZ").timestamp() + ) + artist = entry['master_metadata_album_artist_name'] # hmmm + title = entry['master_metadata_track_name'] + album = entry['master_metadata_album_album_name'] + albumartist = entry['master_metadata_album_artist_name'] + + if played < 30: + yield ('CONFIDENT_SKIP',None,f"{entry} is shorter than 30 seconds, skipping...") + continue + + yield ("CONFIDENT_IMPORT",{ + 'track_title':title, + 'track_artists': artist, + 'track_length': None, + 'scrobble_time': timestamp, + 'scrobble_duration':played, + 'album_name': album, + 'album_artist': albumartist + },'') + except Exception as e: + yield ('FAIL',None,f"{entry} could not be parsed. Scrobble not imported. ({repr(e)})") + continue + + print() + +def parse_spotify(inputf): pth = os.path inputfolder = pth.relpath(pth.dirname(pth.abspath(inputf))) filenames = re.compile(r'endsong_[0-9]+\.json') @@ -180,7 +237,7 @@ def parse_spotify_full(inputf): if len(inputfiles) == 0: print("No files found!") return - + if inputfiles != [inputf]: print("Spotify files should all be imported together to identify duplicates across the whole dataset.") if not ask("Import " + ", ".join(col['yellow'](i) for i in inputfiles) + "?",default=True): From c9ada8c953af15170c23c6d65537fc66b7f95c1f Mon Sep 17 00:00:00 2001 From: krateng Date: Sat, 12 Aug 2023 17:41:31 +0200 Subject: [PATCH 3/5] Added Rockbox import, fix GH-208 --- maloja/proccontrol/tasks/import_scrobbles.py | 43 +++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index 4efd0ce..10febb5 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -33,31 +33,35 @@ def import_scrobbles(inputf): filename = os.path.basename(inputf) - if re.match(".*\.csv",filename): + if re.match(r".*\.csv",filename): typeid,typedesc = "lastfm","Last.fm" importfunc = parse_lastfm - elif re.match("Streaming_History_Audio.+\.json",filename): + elif re.match(r"Streaming_History_Audio.+\.json",filename): typeid,typedesc = "spotify","Spotify" importfunc = parse_spotify_lite - elif re.match("endsong_[0-9]+\.json",filename): + elif re.match(r"endsong_[0-9]+\.json",filename): typeid,typedesc = "spotify","Spotify" importfunc = parse_spotify - elif re.match("StreamingHistory[0-9]+\.json",filename): + elif re.match(r"StreamingHistory[0-9]+\.json",filename): typeid,typedesc = "spotify","Spotify" importfunc = parse_spotify_lite_legacy - elif re.match("maloja_export_[0-9]+\.json",filename): + elif re.match(r"maloja_export_[0-9]+\.json",filename): typeid,typedesc = "maloja","Maloja" importfunc = parse_maloja # username_lb-YYYY-MM-DD.json - elif re.match(".*_lb-[0-9-]+\.json",filename): + elif re.match(r".*_lb-[0-9-]+\.json",filename): typeid,typedesc = "listenbrainz","ListenBrainz" importfunc = parse_listenbrainz + elif re.match(r"\.scrobbler\.log",filename): + typeid,typedesc = "rockbox","Rockbox" + importfunc = parse_rockbox + else: print("File",inputf,"could not be identified as a valid import source.") return result @@ -393,6 +397,33 @@ def parse_listenbrainz(inputf): yield ('FAIL',None,f"{entry} could not be parsed. Scrobble not imported. ({repr(e)})") continue +def parse_rockbox(inputf): + with open(inputf,'r') as inputfd: + for line in inputfd.readlines(): + if line == "#TZ/UNKNOWN": + use_local_time = True + elif line == "#TZ/UTC": + use_local_time = False + line = line.split("#")[0].split("\n")[0] + if line: + try: + artist,album,track,pos,duration,rate,timestamp,track_id, *_ = line.split("\t") + [None] + if rate == 'L': + yield ("CONFIDENT_IMPORT",{ + 'track_title':track, + 'track_artists':artist, + 'track_length':duration, + 'album_name':album, + 'scrobble_time':timestamp, + 'scrobble_duration': None + },'') + else: + yield ('CONFIDENT_SKIP',None,f"{track} at {timestamp} is marked as skipped.") + except Exception as e: + yield ('FAIL',None,f"{line} could not be parsed. Scrobble not imported. ({repr(e)})") + continue + + def parse_maloja(inputf): with open(inputf,'r') as inputfd: From f3555825538e540b96d33ef04d64377a41e54a39 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 13 Aug 2023 17:22:46 +0200 Subject: [PATCH 4/5] Skip invalid scrobbles in Spotify import, fix GH-232 --- maloja/proccontrol/tasks/import_scrobbles.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index 10febb5..5c583f5 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -213,6 +213,10 @@ def parse_spotify_lite(inputf): album = entry['master_metadata_album_album_name'] albumartist = entry['master_metadata_album_artist_name'] + if None in [title,artist]: + yield ('CONFIDENT_SKIP',None,f"{entry} has relevant fields set to null, skipping...") + continue + if played < 30: yield ('CONFIDENT_SKIP',None,f"{entry} is shorter than 30 seconds, skipping...") continue From 139953ddbaeebb8e7513f6702b4e3093e6b3a75f Mon Sep 17 00:00:00 2001 From: krateng Date: Wed, 18 Oct 2023 12:46:35 +0200 Subject: [PATCH 5/5] Simplified env variables --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 7658652..565b97e 100644 --- a/README.md +++ b/README.md @@ -84,11 +84,8 @@ Pull the [latest image](https://hub.docker.com/r/krateng/maloja) or check out th Of note are these settings which should be passed as environmental variables to the container: -#### First run -* `MALOJA_SKIP_SETUP` -- Make the server setup process non-interactive. Maloja will not work properly without this variable set. -* `MALOJA_FORCE_PASSWORD` -- Set an admin password for Maloja. You will need this to log in on first run. - -#### Always +* `MALOJA_SKIP_SETUP` -- Make the server setup process non-interactive. Maloja will not work properly in a container without this variable set. This is done by default in the provided Containerfile. +* `MALOJA_FORCE_PASSWORD` -- Set an admin password for Maloja. You only need this on the first run. * `MALOJA_DATA_DIRECTORY` -- Set the directory in the container where configuration folders/files should be located * Mount a [volume](https://docs.docker.com/engine/reference/builder/#volume) to the specified directory to access these files outside the container (and to make them persistent)