From c1493255b715dd291140224140f50debc450b5df Mon Sep 17 00:00:00 2001 From: badlandspray <92253501+badlandspray@users.noreply.github.com> Date: Sat, 4 Jun 2022 07:31:18 +0000 Subject: [PATCH 01/10] Store album name --- maloja/apis/listenbrainz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 656c11c..595f2aa 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -54,7 +54,7 @@ class Listenbrainz(APIHandler): for listen in payload: try: metadata = listen["track_metadata"] - artiststr, titlestr = metadata["artist_name"], metadata["track_name"] + artiststr, titlestr, albumstr = metadata["artist_name"], metadata["track_name"], metadata["release_name"] additional = metadata.get("additional_info",{}) try: timestamp = int(listen["listened_at"]) @@ -74,6 +74,7 @@ class Listenbrainz(APIHandler): self.scrobble({ 'track_artists':[artiststr], 'track_title':titlestr, + 'album_name':albumstr, 'scrobble_time':timestamp, **extrafields },client=client) From 260c58724851c2cc4054165d8de45e2ce9868395 Mon Sep 17 00:00:00 2001 From: krateng Date: Mon, 6 Jun 2022 18:02:34 +0200 Subject: [PATCH 02/10] Allow minimal listenbrainz payload --- maloja/apis/listenbrainz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 595f2aa..896d6f9 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -54,7 +54,8 @@ class Listenbrainz(APIHandler): for listen in payload: try: metadata = listen["track_metadata"] - artiststr, titlestr, albumstr = metadata["artist_name"], metadata["track_name"], metadata["release_name"] + artiststr, titlestr = metadata["artist_name"], metadata["track_name"] + albumstr = metadata.get("release_name") additional = metadata.get("additional_info",{}) try: timestamp = int(listen["listened_at"]) From d57bf33969dd23903392e5821a9792174e130ff0 Mon Sep 17 00:00:00 2001 From: badlandspray <92253501+badlandspray@users.noreply.github.com> Date: Thu, 9 Jun 2022 17:54:20 +0000 Subject: [PATCH 03/10] Track more additional information --- maloja/apis/listenbrainz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 896d6f9..1c4300b 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -68,7 +68,7 @@ class Listenbrainz(APIHandler): # fields that will not be consumed by regular scrobbling # will go into 'extra' k:additional[k] - for k in ['release_mbid','artist_mbids','recording_mbid','tags'] + for k in ['track_mbid', 'release_mbid','artist_mbids','recording_mbid','tags', 'spotify_id'] if k in additional } @@ -77,6 +77,7 @@ class Listenbrainz(APIHandler): 'track_title':titlestr, 'album_name':albumstr, 'scrobble_time':timestamp, + 'track_length': additional.get("duration"), **extrafields },client=client) From f7251c613c76a598bf3807a718a2b83a61ea3bec Mon Sep 17 00:00:00 2001 From: badlandspray <92253501+badlandspray@users.noreply.github.com> Date: Tue, 5 Jul 2022 08:34:07 +0000 Subject: [PATCH 04/10] Add more fields --- maloja/apis/listenbrainz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 1c4300b..b86c5c9 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -68,7 +68,7 @@ class Listenbrainz(APIHandler): # fields that will not be consumed by regular scrobbling # will go into 'extra' k:additional[k] - for k in ['track_mbid', 'release_mbid','artist_mbids','recording_mbid','tags', 'spotify_id'] + for k in ['track_mbid', 'release_mbid','artist_mbids','recording_mbid','tags', 'origin_url', 'spotify_id', 'music_service', 'music_service_name', 'submission_client'] if k in additional } From 3fd02c167517fff4b7400020db0ee318206aec81 Mon Sep 17 00:00:00 2001 From: badlandspray <92253501+badlandspray@users.noreply.github.com> Date: Fri, 15 Jul 2022 15:18:22 +0000 Subject: [PATCH 05/10] Add release_artist_name and correct duration --- maloja/apis/listenbrainz.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index b86c5c9..4eb3967 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -68,7 +68,7 @@ class Listenbrainz(APIHandler): # fields that will not be consumed by regular scrobbling # will go into 'extra' k:additional[k] - for k in ['track_mbid', 'release_mbid','artist_mbids','recording_mbid','tags', 'origin_url', 'spotify_id', 'music_service', 'music_service_name', 'submission_client'] + for k in ['track_mbid', 'release_mbid', 'release_artist_name', 'artist_mbids','recording_mbid','tags', 'origin_url', 'spotify_id', 'music_service', 'music_service_name', 'submission_client'] if k in additional } @@ -77,7 +77,7 @@ class Listenbrainz(APIHandler): 'track_title':titlestr, 'album_name':albumstr, 'scrobble_time':timestamp, - 'track_length': additional.get("duration"), + 'duration': additional.get("duration"), **extrafields },client=client) From 9ae14da397a235243ebbafcb0a542a002496d863 Mon Sep 17 00:00:00 2001 From: badlandspray <92253501+badlandspray@users.noreply.github.com> Date: Sat, 16 Jul 2022 00:54:54 -0700 Subject: [PATCH 06/10] scrobble_duration key --- maloja/apis/listenbrainz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 4eb3967..5342d90 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -77,7 +77,7 @@ class Listenbrainz(APIHandler): 'track_title':titlestr, 'album_name':albumstr, 'scrobble_time':timestamp, - 'duration': additional.get("duration"), + 'scrobble_duration': additional.get("duration"), **extrafields },client=client) From 2ce2e2f68285ff1f872dc2d3074a18cf64793363 Mon Sep 17 00:00:00 2001 From: Karol Kosek Date: Tue, 16 Aug 2022 19:12:57 +0200 Subject: [PATCH 07/10] Import track lengths from own maloja format It will also be used when importing ListenBrainz files. --- maloja/proccontrol/tasks/import_scrobbles.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index e247521..748d993 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -84,7 +84,7 @@ def import_scrobbles(inputf): "track":{ "artists":scrobble['track_artists'], "title":scrobble['track_title'], - "length":None + "length":scrobble['track_length'], }, "duration":scrobble['scrobble_duration'], "origin":"import:" + typeid, @@ -154,6 +154,7 @@ def parse_spotify_lite(inputf): yield ("CONFIDENT_IMPORT",{ 'track_title':title, 'track_artists': artist, + 'track_length': None, 'scrobble_time': timestamp, 'scrobble_duration':played, 'album_name': None @@ -262,6 +263,7 @@ def parse_spotify_full(inputf): yield (status,{ 'track_title':title, 'track_artists': artist, + 'track_length': None, 'album_name': album, 'scrobble_time': timestamp, 'scrobble_duration':played @@ -294,6 +296,7 @@ def parse_lastfm(inputf): yield ('CONFIDENT_IMPORT',{ 'track_title': title, 'track_artists': artist, + 'track_length': None, 'album_name': album, 'scrobble_time': int(datetime.datetime.strptime( time + '+0000', @@ -318,6 +321,7 @@ def parse_maloja(inputf): yield ('CONFIDENT_IMPORT',{ 'track_title': s['track']['title'], 'track_artists': s['track']['artists'], + 'track_length': s['track']['length'], 'album_name': s['track'].get('album',{}).get('name',''), 'scrobble_time': s['time'], 'scrobble_duration': s['duration'] From 9b10ca4a5d7b547951ced5c7feda9741c0dd9d93 Mon Sep 17 00:00:00 2001 From: Karol Kosek Date: Tue, 16 Aug 2022 19:13:10 +0200 Subject: [PATCH 08/10] Implement importing scrobbles from ListenBrainz Closes: #162 --- README.md | 1 + maloja/proccontrol/tasks/import_scrobbles.py | 27 ++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/README.md b/README.md index 7e8f83b..9d54108 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ If you would like to import your previous scrobbles, use the command `maloja imp * a Last.fm export generated by [benfoxall's website](https://benjaminbenben.com/lastfm-to-csv/) ([GitHub page](https://github.com/benfoxall/lastfm-to-csv)) * an official [Spotify data export file](https://www.spotify.com/us/account/privacy/) +* an official [ListenBrainz export file](https://listenbrainz.org/profile/export/) * the export of another Maloja instance ⚠️ Never import your data while maloja is running. When you need to do import inside docker container start it in shell mode instead and perform import before starting the container as mentioned above. diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index 748d993..b5bf620 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -49,6 +49,11 @@ def import_scrobbles(inputf): typeid,typedesc = "maloja","Maloja" importfunc = parse_maloja + # username_lb-YYYY-MM-DD.json + elif re.match(".*_lb-[0-9-]+\.json",filename): + typeid,typedesc = "listenbrainz","ListenBrainz" + importfunc = parse_listenbrainz + else: print("File",inputf,"could not be identified as a valid import source.") return result @@ -308,6 +313,28 @@ def parse_lastfm(inputf): yield ('FAIL',None,f"{row} (Line {line}) could not be parsed. Scrobble not imported. ({repr(e)})") continue +def parse_listenbrainz(inputf): + + with open(inputf,'r') as inputfd: + data = json.load(inputfd) + + for entry in data: + + try: + track_metadata = entry['track_metadata'] + additional_info = track_metadata.get('additional_info', {}) + + yield ("CONFIDENT_IMPORT",{ + 'track_title': track_metadata['track_name'], + 'track_artists': additional_info.get('artist_names') or track_metadata['artist_name'], + 'track_length': int(additional_info.get('duration_ms', 0) / 1000) or additional_info.get('duration'), + 'album_name': track_metadata.get('release_name'), + 'scrobble_time': entry['listened_at'], + 'scrobble_duration': None, + },'') + except Exception as e: + yield ('FAIL',None,f"{entry} could not be parsed. Scrobble not imported. ({repr(e)})") + continue def parse_maloja(inputf): From d5f2c254f3a9a7e9ea5a8799ac049d2a96864a41 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 11 Sep 2022 20:58:37 +0200 Subject: [PATCH 09/10] Fix field name for track length --- maloja/apis/listenbrainz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 5342d90..db0053c 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -77,7 +77,7 @@ class Listenbrainz(APIHandler): 'track_title':titlestr, 'album_name':albumstr, 'scrobble_time':timestamp, - 'scrobble_duration': additional.get("duration"), + 'track_length': additional.get("duration"), **extrafields },client=client) From 91750db8ac0a73db66609186c7eb993e4512cd04 Mon Sep 17 00:00:00 2001 From: krateng Date: Sun, 11 Sep 2022 21:01:21 +0200 Subject: [PATCH 10/10] Reduce stored extra info from Listenbrainz API --- maloja/apis/listenbrainz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index db0053c..234048c 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -68,7 +68,7 @@ class Listenbrainz(APIHandler): # fields that will not be consumed by regular scrobbling # will go into 'extra' k:additional[k] - for k in ['track_mbid', 'release_mbid', 'release_artist_name', 'artist_mbids','recording_mbid','tags', 'origin_url', 'spotify_id', 'music_service', 'music_service_name', 'submission_client'] + for k in ['track_mbid', 'release_mbid', 'artist_mbids','recording_mbid','tags'] if k in additional }