diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 0fd9451..d2c00ab 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1 @@ -custom: ["https://flattr.com/@Krateng", "https://paypal.me/krateng"] +custom: ["https://paypal.me/krateng"] diff --git a/Containerfile b/Containerfile index 358efe2..55057d8 100644 --- a/Containerfile +++ b/Containerfile @@ -12,6 +12,7 @@ COPY --chown=abc:abc ./requirements.txt ./requirements.txt # it may be possible to decrease image size slightly by using build stage and # copying all site-packages to runtime stage but the image is already pretty small RUN \ + echo "" && \ echo "**** install build packages ****" && \ apk add --no-cache --virtual=build-deps \ gcc \ @@ -23,18 +24,22 @@ RUN \ libc-dev \ py3-pip \ linux-headers && \ + echo "" && \ echo "**** install runtime packages ****" && \ apk add --no-cache \ python3 \ py3-lxml \ tzdata && \ + echo "" && \ echo "**** install pip dependencies ****" && \ python3 -m ensurepip && \ pip3 install -U --no-cache-dir \ pip \ wheel && \ + echo "" && \ echo "**** install maloja requirements ****" && \ pip3 install --no-cache-dir -r requirements.txt && \ + echo "" && \ echo "**** cleanup ****" && \ apk del --purge \ build-deps && \ @@ -47,6 +52,7 @@ RUN \ COPY --chown=abc:abc . . RUN \ + echo "" && \ echo "**** install maloja ****" && \ apk add --no-cache --virtual=install-deps \ py3-pip && \ diff --git a/README.md b/README.md index bdc8493..d1fb4dd 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,7 @@ [![](https://img.shields.io/pypi/v/malojaserver?label=PyPI&style=for-the-badge&logo=pypi&logoColor=white)](https://pypi.org/project/malojaserver/) [![](https://img.shields.io/docker/v/krateng/maloja?label=Dockerhub&style=for-the-badge&logo=docker&logoColor=white)](https://hub.docker.com/r/krateng/maloja) -[![](https://img.shields.io/pypi/l/malojaserver?style=for-the-badge)](https://github.com/krateng/maloja/blob/master/LICENSE) -[![](https://img.shields.io/codeclimate/maintainability/krateng/maloja?style=for-the-badge)](https://codeclimate.com/github/krateng/maloja) - -Simple self-hosted music scrobble database to create personal listening statistics. No recommendations, no social network, no nonsense. +Simple self-hosted music scrobble database to create personal listening statistics. ![screenshot](https://raw.githubusercontent.com/krateng/maloja/master/screenshot.png) @@ -18,9 +15,9 @@ You can check [my own Maloja page](https://maloja.krateng.ch) as an example inst * [Features](#features) * [How to install](#how-to-install) * [Requirements](#requirements) + * [Docker / Podman](#docker--podman) * [PyPI](#pypi) * [From Source](#from-source) - * [Docker / Podman](#docker--podman) * [Extras](#extras) * [How to use](#how-to-use) * [Basic control](#basic-control) @@ -47,10 +44,46 @@ You can check [my own Maloja page](https://maloja.krateng.ch) as an example inst Maloja should run on any x86 or ARM machine that runs Python. -I can support you with issues best if you use **Alpine Linux**. +It is highly recommended to use **Docker** or **Podman**. Your CPU should have a single core passmark score of at the very least 1500. 500 MB RAM should give you a decent experience, but performance will benefit greatly from up to 2 GB. +### Docker / Podman + +Pull the [latest image](https://hub.docker.com/r/krateng/maloja) or check out the repository and use the included Containerfile. + +Of note are these settings which should be passed as environmental variables to the container: + +* `MALOJA_SKIP_SETUP` -- Make the server setup process non-interactive. Maloja will not work properly in a container without this variable set. This is done by default in the provided Containerfile. +* `MALOJA_FORCE_PASSWORD` -- Set an admin password for Maloja. You only need this on the first run. +* `MALOJA_DATA_DIRECTORY` -- Set the directory in the container where configuration folders/files should be located + * Mount a [volume](https://docs.docker.com/engine/reference/builder/#volume) to the specified directory to access these files outside the container (and to make them persistent) + +You must publish a port on your host machine to bind to the container's web port (default 42010). The container uses IPv4 per default. + +An example of a minimum run configuration to access maloja via `localhost:42010`: + +```console + docker run -p 42010:42010 -v $PWD/malojadata:/mljdata -e MALOJA_DATA_DIRECTORY=/mljdata krateng/maloja +``` + +#### Linux Host + +**NOTE:** If you are using [rootless containers with Podman](https://developers.redhat.com/blog/2020/09/25/rootless-containers-with-podman-the-basics#why_podman_) this DOES NOT apply to you. + +If you are running Docker on a **Linux Host** you should specify `user:group` ids of the user who owns the folder on the host machine bound to `MALOJA_DATA_DIRECTORY` in order to avoid [docker file permission problems.](https://ikriv.com/blog/?p=4698) These can be specified using the [environmental variables **PUID** and **PGID**.](https://docs.linuxserver.io/general/understanding-puid-and-pgid) + +To get the UID and GID for the current user run these commands from a terminal: + +* `id -u` -- prints UID (EX `1000`) +* `id -g` -- prints GID (EX `1001`) + +The modified run command with these variables would look like: + +```console + docker run -e PUID=1000 -e PGID=1001 -p 42010:42010 -v $PWD/malojadata:/mljdata -e MALOJA_DATA_DIRECTORY=/mljdata krateng/maloja +``` + ### PyPI You can install Maloja with @@ -78,41 +111,6 @@ Then install all the requirements and build the package, e.g.: pip install . ``` -### Docker / Podman - -Pull the [latest image](https://hub.docker.com/r/krateng/maloja) or check out the repository and use the included Containerfile. - -Of note are these settings which should be passed as environmental variables to the container: - -* `MALOJA_SKIP_SETUP` -- Make the server setup process non-interactive. Maloja will not work properly in a container without this variable set. This is done by default in the provided Containerfile. -* `MALOJA_FORCE_PASSWORD` -- Set an admin password for Maloja. You only need this on the first run. -* `MALOJA_DATA_DIRECTORY` -- Set the directory in the container where configuration folders/files should be located - * Mount a [volume](https://docs.docker.com/engine/reference/builder/#volume) to the specified directory to access these files outside the container (and to make them persistent) - -You must publish a port on your host machine to bind to the container's web port (default 42010). The container uses IPv4 per default. - -An example of a minimum run configuration to access maloja via `localhost:42010`: - -```console - docker run -p 42010:42010 -v $PWD/malojadata:/mljdata -e MALOJA_DATA_DIRECTORY=/mljdata krateng/maloja -``` - -#### Linux Host - -**NOTE:** If you are using [rootless containers with Podman](https://developers.redhat.com/blog/2020/09/25/rootless-containers-with-podman-the-basics#why_podman_) this DOES NOT apply to you. - -If you are running Docker on a **Linux Host** you should specify `user:group` ids of the user who owns the folder on the host machine bound to `MALOJA_DATA_DIRECTORY` in order to avoid [docker file permission problems.](https://ikriv.com/blog/?p=4698) These can be specified using the [environmental variables **PUID** and **PGID**.](https://docs.linuxserver.io/general/understanding-puid-and-pgid) - -To get the UID and GID for the current user run these commands from a terminal: - -* `id -u` -- prints UID (EX `1000`) -* `id -g` -- prints GID (EX `1001`) - -The modified run command with these variables would look like: - -```console - docker run -e PUID=1000 -e PGID=1001 -p 42010:42010 -v $PWD/malojadata:/mljdata -e MALOJA_DATA_DIRECTORY=/mljdata krateng/maloja -``` ### Extras @@ -120,31 +118,13 @@ The modified run command with these variables would look like: * Put your server behind a reverse proxy for SSL encryption. Make sure that you're proxying to the IPv6 or IPv4 address according to your settings. -* You can set up a cronjob to start your server on system boot, and potentially restart it on a regular basis: - -``` -@reboot sleep 15 && maloja start -42 0 7 * * maloja restart -``` - ## How to use ### Basic control -Start and stop the server in the background with - -```console - maloja start - maloja stop - maloja restart -``` - -If you need to run the server in the foreground, use - -```console - maloja run -``` +When not running in a container, you can run the application with `maloja run`. You can also run it in the background with +`maloja start` and `maloja stop`, but this might not be supported in the future. ### Data @@ -164,7 +144,6 @@ If you would like to import your previous scrobbles, use the command `maloja imp maloja import my_last_fm_export.csv ``` ---- To backup your data, run `maloja backup`, optional with `--include_images`. diff --git a/dev/releases/3.1.yml b/dev/releases/3.1.yml index 7187b5b..a417691 100644 --- a/dev/releases/3.1.yml +++ b/dev/releases/3.1.yml @@ -41,6 +41,3 @@ minor_release_name: "Soyeon" - "[Bugfix] Fixed image display on Safari" - "[Bugfix] Fixed entity editing on Firefox" - "[Bugfix] Made compatibile with SQLAlchemy 2.0" -upcoming: - notes: - - "[Bugfix] Fixed configuration of time format" diff --git a/dev/releases/3.2.yml b/dev/releases/3.2.yml new file mode 100644 index 0000000..d6d27db --- /dev/null +++ b/dev/releases/3.2.yml @@ -0,0 +1,19 @@ +minor_release_name: "Nicole" +3.2.0: + notes: + - "[Architecture] Switched to linuxserver.io container base image" + - "[Architecture] Reworked image handling" + - "[Architecture] Removed pre-calculated stats" + - "[Feature] Added support for albums" + - "[Feature] New start page" + - "[Feature] Added UI for track-artist, track-album and album-artist association" + - "[Feature] Added inline UI for association and merging in chart lists" + - "[Feature] Added UI selector for including associated artists" + - "[Feature] Added UI distinction for associated scrobbles in chart bars" + - "[Performance] Improved image rendering" + - "[Performance] Optimized several database calls" + - "[Bugfix] Fixed configuration of time format" + - "[Bugfix] Fixed search on manual scrobble page" + - "[Bugfix] Disabled DB maintenance while not running main server" + - "[Bugfix] Removed some nonsensical ephemereal database entry creations" + - "[Bugfix] Fixed API endpoint for track charts with no artist provided" diff --git a/dev/testing/Maloja.postman_collection.json b/dev/testing/Maloja.postman_collection.json index 7ca23a4..852bd92 100644 --- a/dev/testing/Maloja.postman_collection.json +++ b/dev/testing/Maloja.postman_collection.json @@ -189,7 +189,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"key\": \"{{api_key}}\",\n \"artist\": \"{{data.artist1}}\",\n \"title\": \"{{data.title1}}\"\n}" + "raw": "{\n \"key\": \"{{api_key}}\",\n \"artist\": \"{{data.artist1}}\",\n \"title\": \"{{data.title1}}\",\n \"album\": \"{{data.album}}\",\n \"albumartists\":[\n \"{{data.artist1}}\",\n \"{{data.artist3}}\"\n ]\n}" }, "url": { "raw": "{{url}}/apis/mlj_1/newscrobble", @@ -219,7 +219,7 @@ ], "body": { "mode": "raw", - "raw": "{\n \"key\": \"{{api_key}}\",\n \"artists\": [\"{{data.artist1}}\",\"{{data.artist2}}\"],\n \"title\": \"{{data.title1}}\"\n}" + "raw": "{\n \"key\": \"{{api_key}}\",\n \"artists\": [\"{{data.artist1}}\",\"{{data.artist2}}\"],\n \"title\": \"{{data.title1}}\",\n \"album\": \"{{data.album}}\",\n \"albumartists\":[\n \"{{data.artist1}}\",\n \"{{data.artist3}}\"\n ]\n}" }, "url": { "raw": "{{url}}/apis/mlj_1/newscrobble", @@ -867,6 +867,11 @@ "key": "data.title3", "value": "One in a Million" }, + { + "key": "data.album", + "value": "The Epic Collection", + "type": "default" + }, { "key": "data.timestamp1", "value": "" diff --git a/maloja/__main__.py b/maloja/__main__.py index 5e6d4de..5c4692f 100644 --- a/maloja/__main__.py +++ b/maloja/__main__.py @@ -30,13 +30,13 @@ def print_header_info(): def get_instance(): try: - return int(subprocess.check_output(["pidof","maloja"])) + return int(subprocess.check_output(["pgrep","-x","maloja"])) except Exception: return None def get_instance_supervisor(): try: - return int(subprocess.check_output(["pidof","maloja_supervisor"])) + return int(subprocess.check_output(["pgrep","-x","maloja_supervisor"])) except Exception: return None @@ -130,7 +130,7 @@ def run_supervisor(): def debug(): os.environ["MALOJA_DEV_MODE"] = 'true' conf.malojaconfig.load_environment() - direct() + run_server() def print_info(): print_header_info() @@ -141,14 +141,15 @@ def print_info(): print(col['lightblue']("Timezone: "),f"UTC{conf.malojaconfig['timezone']:+d}") print() try: - import pkg_resources + from importlib.metadata import distribution for pkg in ("sqlalchemy","waitress","bottle","doreah","jinja2"): - print(col['cyan'] (f"{pkg}:".ljust(13)),pkg_resources.get_distribution(pkg).version) + print(col['cyan'] (f"{pkg}:".ljust(13)),distribution(pkg).version) except ImportError: + raise print("Could not determine dependency versions.") print() -@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images'],shield=True) +@mainfunction({"l":"level","v":"version","V":"version"},flags=['version','include_images','prefer_existing'],shield=True) def main(*args,**kwargs): actions = { @@ -166,12 +167,13 @@ def main(*args,**kwargs): "generate":generate.generate_scrobbles, # maloja generate 400 "export":tasks.export, # maloja export "apidebug":apidebug.run, # maloja apidebug + "parsealbums":tasks.parse_albums, # maloja parsealbums --strategy majority # aux "info":print_info } if "version" in kwargs: - print(info.VERSION) + print(pkginfo.VERSION) return True else: try: diff --git a/maloja/apis/listenbrainz.py b/maloja/apis/listenbrainz.py index 234048c..cb15b89 100644 --- a/maloja/apis/listenbrainz.py +++ b/maloja/apis/listenbrainz.py @@ -75,7 +75,7 @@ class Listenbrainz(APIHandler): self.scrobble({ 'track_artists':[artiststr], 'track_title':titlestr, - 'album_name':albumstr, + 'album_title':albumstr, 'scrobble_time':timestamp, 'track_length': additional.get("duration"), **extrafields diff --git a/maloja/apis/native_v1.py b/maloja/apis/native_v1.py index 41fa636..aea7710 100644 --- a/maloja/apis/native_v1.py +++ b/maloja/apis/native_v1.py @@ -2,10 +2,12 @@ import os import math import traceback -from bottle import response, static_file, request, FormsDict +from bottle import response, static_file, FormsDict + +from inspect import signature from doreah.logging import log -from doreah.auth import authenticated_api, authenticated_api_with_alternate, authenticated_function +from doreah.auth import authenticated_function # nimrodel API from nimrodel import EAPI as API @@ -18,7 +20,7 @@ from ..pkg_global.conf import malojaconfig, data_dir from ..__pkginfo__ import VERSION -from ..malojauri import uri_to_internal, compose_querystring, internal_to_uri +from ..malojauri import uri_to_internal, compose_querystring, internal_to_uri, create_uri from .. import images from ._apikeys import apikeystore, api_key_correct @@ -72,6 +74,14 @@ errors = { 'desc':"The database is being upgraded. Please try again later." } }), + database.exceptions.EntityDoesNotExist: lambda e: (404,{ + "status":"error", + "error":{ + 'type':'entity_does_not_exist', + 'value':e.entitydict, + 'desc':"This entity does not exist in the database." + } + }), images.MalformedB64: lambda e: (400,{ "status":"failure", "error":{ @@ -91,6 +101,8 @@ errors = { }) } + +# decorator to catch exceptions and return proper json responses def catch_exceptions(func): def protector(*args,**kwargs): try: @@ -105,9 +117,11 @@ def catch_exceptions(func): protector.__doc__ = func.__doc__ protector.__annotations__ = func.__annotations__ + protector.__name__ = f"EXCPR_{func.__name__}" return protector +# decorator to expand the docstring with common arguments for the API explorer. DOESNT WRAP def add_common_args_to_docstring(filterkeys=False,limitkeys=False,delimitkeys=False,amountkeys=False): def decorator(func): timeformats = "Possible formats include '2022', '2022/08', '2022/08/01', '2022/W42', 'today', 'thismonth', 'monday', 'august'" @@ -141,6 +155,64 @@ def add_common_args_to_docstring(filterkeys=False,limitkeys=False,delimitkeys=Fa return decorator +# decorator to take the URI keys and convert them into internal keys +def convert_kwargs(func): + + #params = tuple(p for p in signature(func).parameters) + + def wrapper(*args,albumartist:Multi[str]=[],trackartist:Multi[str]=[],**kwargs): + + kwargs = FormsDict(kwargs) + for a in albumartist: + kwargs.append("albumartist",a) + for a in trackartist: + kwargs.append("trackartist",a) + + k_filter, k_limit, k_delimit, k_amount, k_special = uri_to_internal(kwargs,api=True) + + try: + return func(*args,k_filter=k_filter, k_limit=k_limit, k_delimit=k_delimit, k_amount=k_amount) + except TypeError: + return func(*args,k_filter=k_filter, k_limit=k_limit, k_delimit=k_delimit, k_amount=k_amount,k_special=k_special) + # TODO: ....really? + + wrapper.__doc__ = func.__doc__ + wrapper.__name__ = f"CVKWA_{func.__name__}" + return wrapper + + +# decorator to add pagination info to endpoints (like links to other pages) +# this expects already converted uri args!!! +def add_pagination(endpoint,filterkeys=False,limitkeys=False,delimitkeys=False): + + def decorator(func): + def wrapper(*args,k_filter, k_limit, k_delimit, k_amount): + + keydicts = [] + if filterkeys: keydicts.append(k_filter) + if limitkeys: keydicts.append(k_limit) + if delimitkeys: keydicts.append(k_delimit) + keydicts.append(k_amount) + + + result = func(*args,k_filter=k_filter, k_limit=k_limit, k_delimit=k_delimit, k_amount=k_amount) + + result['pagination'] = { + 'page': k_amount['page'], + 'perpage': k_amount['perpage'] if (k_amount['perpage'] is not math.inf) else None, + 'next_page': create_uri(api.pathprefix + '/' + endpoint,*keydicts,{'page':k_amount['page']+1}) if len(result.get('list',[]))==k_amount['perpage'] else None, + 'prev_page': create_uri(api.pathprefix + '/' + endpoint,*keydicts,{'page':k_amount['page']-1}) if k_amount['page'] > 0 else None + } + + return result + + wrapper.__doc__ = func.__doc__ + wrapper.__annotations__ = func.__annotations__ + wrapper.__name__ = f"PGNAT_{func.__name__}" + return wrapper + + return decorator + @api.get("test") @catch_exceptions @@ -194,20 +266,22 @@ def server_info(): @api.get("scrobbles") @catch_exceptions @add_common_args_to_docstring(filterkeys=True,limitkeys=True,amountkeys=True) -def get_scrobbles_external(**keys): +@convert_kwargs +@add_pagination("scrobbles",filterkeys=True,limitkeys=True) +def get_scrobbles_external(k_filter, k_limit, k_delimit, k_amount): """Returns a list of scrobbles. :return: list (List) :rtype: Dictionary """ - k_filter, k_time, _, k_amount, _ = uri_to_internal(keys,api=True) - ckeys = {**k_filter, **k_time, **k_amount} + ckeys = {**k_filter, **k_limit, **k_amount} result = database.get_scrobbles(**ckeys) - offset = (k_amount.get('page') * k_amount.get('perpage')) if k_amount.get('perpage') is not math.inf else 0 - result = result[offset:] - if k_amount.get('perpage') is not math.inf: result = result[:k_amount.get('perpage')] + # this should now all be served by the inner function + #offset = (k_amount.get('page') * k_amount.get('perpage')) if k_amount.get('perpage') is not math.inf else 0 + #result = result[offset:] + #if k_amount.get('perpage') is not math.inf: result = result[:k_amount.get('perpage')] return { "status":"ok", @@ -218,15 +292,15 @@ def get_scrobbles_external(**keys): @api.get("numscrobbles") @catch_exceptions @add_common_args_to_docstring(filterkeys=True,limitkeys=True,amountkeys=True) -def get_scrobbles_num_external(**keys): +@convert_kwargs +def get_scrobbles_num_external(k_filter, k_limit, k_delimit, k_amount): """Returns amount of scrobbles. :return: amount (Integer) :rtype: Dictionary """ - k_filter, k_time, _, k_amount, _ = uri_to_internal(keys) - ckeys = {**k_filter, **k_time, **k_amount} + ckeys = {**k_filter, **k_limit, **k_amount} result = database.get_scrobbles_num(**ckeys) return { @@ -235,19 +309,18 @@ def get_scrobbles_num_external(**keys): } - @api.get("tracks") @catch_exceptions @add_common_args_to_docstring(filterkeys=True) -def get_tracks_external(**keys): - """Returns all tracks (optionally of an artist). +@convert_kwargs +def get_tracks_external(k_filter, k_limit, k_delimit, k_amount): + """Returns all tracks (optionally of an artist or on an album). :return: list (List) :rtype: Dictionary """ - k_filter, _, _, _, _ = uri_to_internal(keys,forceArtist=True) - ckeys = {**k_filter} + ckeys = {**k_filter} result = database.get_tracks(**ckeys) return { @@ -256,15 +329,16 @@ def get_tracks_external(**keys): } - @api.get("artists") @catch_exceptions @add_common_args_to_docstring() -def get_artists_external(): +@convert_kwargs +def get_artists_external(k_filter, k_limit, k_delimit, k_amount): """Returns all artists. :return: list (List) :rtype: Dictionary""" + result = database.get_artists() return { @@ -273,20 +347,36 @@ def get_artists_external(): } +@api.get("albums") +@catch_exceptions +@add_common_args_to_docstring(filterkeys=True) +@convert_kwargs +def get_albums_external(k_filter, k_limit, k_delimit, k_amount): + """Returns all albums (optionally of an artist). + :return: list (List) + :rtype: Dictionary""" + + ckeys = {**k_filter} + result = database.get_albums(**ckeys) + + return { + "status":"ok", + "list":result + } @api.get("charts/artists") @catch_exceptions @add_common_args_to_docstring(limitkeys=True) -def get_charts_artists_external(**keys): +@convert_kwargs +def get_charts_artists_external(k_filter, k_limit, k_delimit, k_amount): """Returns artist charts :return: list (List) :rtype: Dictionary""" - _, k_time, _, _, _ = uri_to_internal(keys) - ckeys = {**k_time} + ckeys = {**k_limit} result = database.get_charts_artists(**ckeys) return { @@ -295,18 +385,17 @@ def get_charts_artists_external(**keys): } - @api.get("charts/tracks") @catch_exceptions @add_common_args_to_docstring(filterkeys=True,limitkeys=True) -def get_charts_tracks_external(**keys): +@convert_kwargs +def get_charts_tracks_external(k_filter, k_limit, k_delimit, k_amount): """Returns track charts :return: list (List) :rtype: Dictionary""" - k_filter, k_time, _, _, _ = uri_to_internal(keys,forceArtist=True) - ckeys = {**k_filter, **k_time} + ckeys = {**k_filter, **k_limit} result = database.get_charts_tracks(**ckeys) return { @@ -315,19 +404,36 @@ def get_charts_tracks_external(**keys): } +@api.get("charts/albums") +@catch_exceptions +@add_common_args_to_docstring(filterkeys=True,limitkeys=True) +@convert_kwargs +def get_charts_albums_external(k_filter, k_limit, k_delimit, k_amount): + """Returns album charts + + :return: list (List) + :rtype: Dictionary""" + + ckeys = {**k_filter, **k_limit} + result = database.get_charts_albums(**ckeys) + + return { + "status":"ok", + "list":result + } @api.get("pulse") @catch_exceptions @add_common_args_to_docstring(filterkeys=True,limitkeys=True,delimitkeys=True,amountkeys=True) -def get_pulse_external(**keys): +@convert_kwargs +def get_pulse_external(k_filter, k_limit, k_delimit, k_amount): """Returns amounts of scrobbles in specified time frames :return: list (List) :rtype: Dictionary""" - k_filter, k_time, k_internal, k_amount, _ = uri_to_internal(keys) - ckeys = {**k_filter, **k_time, **k_internal, **k_amount} + ckeys = {**k_filter, **k_limit, **k_delimit, **k_amount} results = database.get_pulse(**ckeys) return { @@ -336,19 +442,17 @@ def get_pulse_external(**keys): } - - @api.get("performance") @catch_exceptions @add_common_args_to_docstring(filterkeys=True,limitkeys=True,delimitkeys=True,amountkeys=True) -def get_performance_external(**keys): +@convert_kwargs +def get_performance_external(k_filter, k_limit, k_delimit, k_amount): """Returns artist's or track's rank in specified time frames :return: list (List) :rtype: Dictionary""" - k_filter, k_time, k_internal, k_amount, _ = uri_to_internal(keys) - ckeys = {**k_filter, **k_time, **k_internal, **k_amount} + ckeys = {**k_filter, **k_limit, **k_delimit, **k_amount} results = database.get_performance(**ckeys) return { @@ -362,14 +466,14 @@ def get_performance_external(**keys): @api.get("top/artists") @catch_exceptions @add_common_args_to_docstring(limitkeys=True,delimitkeys=True) -def get_top_artists_external(**keys): +@convert_kwargs +def get_top_artists_external(k_filter, k_limit, k_delimit, k_amount): """Returns respective number 1 artists in specified time frames :return: list (List) :rtype: Dictionary""" - _, k_time, k_internal, _, _ = uri_to_internal(keys) - ckeys = {**k_time, **k_internal} + ckeys = {**k_limit, **k_delimit} results = database.get_top_artists(**ckeys) return { @@ -378,22 +482,19 @@ def get_top_artists_external(**keys): } - - @api.get("top/tracks") @catch_exceptions @add_common_args_to_docstring(limitkeys=True,delimitkeys=True) -def get_top_tracks_external(**keys): +@convert_kwargs +def get_top_tracks_external(k_filter, k_limit, k_delimit, k_amount): """Returns respective number 1 tracks in specified time frames :return: list (List) :rtype: Dictionary""" - _, k_time, k_internal, _, _ = uri_to_internal(keys) - ckeys = {**k_time, **k_internal} - - # IMPLEMENT THIS FOR TOP TRACKS OF ARTIST AS WELL? + ckeys = {**k_limit, **k_delimit} results = database.get_top_tracks(**ckeys) + # IMPLEMENT THIS FOR TOP TRACKS OF ARTIST/ALBUM AS WELL? return { "status":"ok", @@ -401,17 +502,36 @@ def get_top_tracks_external(**keys): } +@api.get("top/albums") +@catch_exceptions +@add_common_args_to_docstring(limitkeys=True,delimitkeys=True) +@convert_kwargs +def get_top_albums_external(k_filter, k_limit, k_delimit, k_amount): + """Returns respective number 1 albums in specified time frames + + :return: list (List) + :rtype: Dictionary""" + + ckeys = {**k_limit, **k_delimit} + results = database.get_top_albums(**ckeys) + # IMPLEMENT THIS FOR TOP ALBUMS OF ARTIST AS WELL? + + return { + "status":"ok", + "list":results + } @api.get("artistinfo") @catch_exceptions @add_common_args_to_docstring(filterkeys=True) -def artist_info_external(**keys): +@convert_kwargs +def artist_info_external(k_filter, k_limit, k_delimit, k_amount): """Returns information about an artist :return: artist (String), scrobbles (Integer), position (Integer), associated (List), medals (Mapping), topweeks (Integer) :rtype: Dictionary""" - k_filter, _, _, _, _ = uri_to_internal(keys,forceArtist=True) + ckeys = {**k_filter} return database.artist_info(**ckeys) @@ -421,21 +541,31 @@ def artist_info_external(**keys): @api.get("trackinfo") @catch_exceptions @add_common_args_to_docstring(filterkeys=True) -def track_info_external(artist:Multi[str]=[],**keys): +@convert_kwargs +def track_info_external(k_filter, k_limit, k_delimit, k_amount): """Returns information about a track :return: track (Mapping), scrobbles (Integer), position (Integer), medals (Mapping), certification (String), topweeks (Integer) :rtype: Dictionary""" - # transform into a multidict so we can use our nomral uri_to_internal function - keys = FormsDict(keys) - for a in artist: - keys.append("artist",a) - k_filter, _, _, _, _ = uri_to_internal(keys,forceTrack=True) - ckeys = {**k_filter} + ckeys = {**k_filter} return database.track_info(**ckeys) +@api.get("albuminfo") +@catch_exceptions +@add_common_args_to_docstring(filterkeys=True) +@convert_kwargs +def album_info_external(k_filter, k_limit, k_delimit, k_amount): + """Returns information about an album + + :return: album (Mapping), scrobbles (Integer), position (Integer), medals (Mapping), certification (String), topweeks (Integer) + :rtype: Dictionary""" + + ckeys = {**k_filter} + return database.album_info(**ckeys) + + @api.post("newscrobble") @authenticated_function(alternate=api_key_correct,api=True,pass_auth_result_as='auth_result') @catch_exceptions @@ -444,7 +574,7 @@ def post_scrobble( artists:list=[], title:str="", album:str=None, - albumartists:list=[], + albumartists:list=None, duration:int=None, length:int=None, time:int=None, @@ -470,7 +600,7 @@ def post_scrobble( rawscrobble = { 'track_artists':(artist or []) + artists, 'track_title':title, - 'album_name':album, + 'album_title':album, 'album_artists':albumartists, 'scrobble_duration':duration, 'track_length':length, @@ -478,7 +608,7 @@ def post_scrobble( } # for logging purposes, don't pass values that we didn't actually supply - rawscrobble = {k:rawscrobble[k] for k in rawscrobble if rawscrobble[k]} + rawscrobble = {k:rawscrobble[k] for k in rawscrobble if rawscrobble[k] is not None} # [] should be passed result = database.incoming_scrobble( @@ -494,19 +624,23 @@ def post_scrobble( 'artists':result['track']['artists'], 'title':result['track']['title'] }, - 'desc':f"Scrobbled {result['track']['title']} by {', '.join(result['track']['artists'])}" + 'desc':f"Scrobbled {result['track']['title']} by {', '.join(result['track']['artists'])}", + 'warnings':[] } if extra_kwargs: - responsedict['warnings'] = [ + responsedict['warnings'] += [ {'type':'invalid_keyword_ignored','value':k, 'desc':"This key was not recognized by the server and has been discarded."} for k in extra_kwargs ] if artist and artists: - responsedict['warnings'] = [ + responsedict['warnings'] += [ {'type':'mixed_schema','value':['artist','artists'], 'desc':"These two fields are meant as alternative methods to submit information. Use of both is discouraged, but works at the moment."} ] + + if len(responsedict['warnings']) == 0: del responsedict['warnings'] + return responsedict @@ -515,21 +649,18 @@ def post_scrobble( @api.post("addpicture") @authenticated_function(alternate=api_key_correct,api=True) @catch_exceptions -def add_picture(b64,artist:Multi=[],title=None): - """Uploads a new image for an artist or track. +@convert_kwargs +def add_picture(k_filter, k_limit, k_delimit, k_amount, k_special): + """Uploads a new image for an artist, album or track. param string b64: Base 64 representation of the image - param string artist: Artist name. Can be supplied multiple times for tracks with multiple artists. - param string title: Title of the track. Optional. """ - keys = FormsDict() - for a in artist: - keys.append("artist",a) - if title is not None: keys.append("title",title) - k_filter, _, _, _, _ = uri_to_internal(keys) + + if "associated" in k_filter: del k_filter["associated"] if "track" in k_filter: k_filter = k_filter["track"] - url = images.set_image(b64,**k_filter) + elif "album" in k_filter: k_filter = k_filter["album"] + url = images.set_image(k_special['b64'],**k_filter) return { 'status': 'success', @@ -586,6 +717,7 @@ def search(**keys): artists = database.db_search(query,type="ARTIST") tracks = database.db_search(query,type="TRACK") + albums = database.db_search(query,type="ALBUM") @@ -593,6 +725,7 @@ def search(**keys): # also, shorter is better (because longer titles would be easier to further specify) artists.sort(key=lambda x: ((0 if x.lower().startswith(query) else 1 if " " + query in x.lower() else 2),len(x))) tracks.sort(key=lambda x: ((0 if x["title"].lower().startswith(query) else 1 if " " + query in x["title"].lower() else 2),len(x["title"]))) + albums.sort(key=lambda x: ((0 if x["albumtitle"].lower().startswith(query) else 1 if " " + query in x["albumtitle"].lower() else 2),len(x["albumtitle"]))) # add links artists_result = [] @@ -613,7 +746,22 @@ def search(**keys): } tracks_result.append(result) - return {"artists":artists_result[:max_],"tracks":tracks_result[:max_]} + albums_result = [] + for al in albums: + result = { + 'album': al, + 'link': "/album?" + compose_querystring(internal_to_uri({"album":al})), + 'image': images.get_album_image(al) + } + mutable_result = result.copy() + mutable_result['album'] = result['album'].copy() + if not mutable_result['album']['artists']: mutable_result['album']['displayArtist'] = malojaconfig["DEFAULT_ALBUM_ARTIST"] + # we don't wanna actually mutate the dict here because this is in the cache + # TODO: This should be globally solved!!!!! immutable dicts with mutable overlays??? + # this is a major flaw in the architecture! + albums_result.append(mutable_result) + + return {"artists":artists_result[:max_],"tracks":tracks_result[:max_],"albums":albums_result[:max_]} @api.post("newrule") @@ -708,6 +856,16 @@ def edit_track(id,title): "status":"success" } +@api.post("edit_album") +@authenticated_function(api=True) +@catch_exceptions +def edit_album(id,albumtitle): + """Internal Use Only""" + result = database.edit_album(id,{'albumtitle':albumtitle}) + return { + "status":"success" + } + @api.post("merge_tracks") @authenticated_function(api=True) @@ -716,7 +874,8 @@ def merge_tracks(target_id,source_ids): """Internal Use Only""" result = database.merge_tracks(target_id,source_ids) return { - "status":"success" + "status":"success", + "desc":f"{', '.join(src['title'] for src in result['sources'])} were merged into {result['target']['title']}" } @api.post("merge_artists") @@ -726,9 +885,57 @@ def merge_artists(target_id,source_ids): """Internal Use Only""" result = database.merge_artists(target_id,source_ids) return { - "status":"success" + "status":"success", + "desc":f"{', '.join(src for src in result['sources'])} were merged into {result['target']}" } +@api.post("merge_albums") +@authenticated_function(api=True) +@catch_exceptions +def merge_artists(target_id,source_ids): + """Internal Use Only""" + result = database.merge_albums(target_id,source_ids) + return { + "status":"success", + "desc":f"{', '.join(src['albumtitle'] for src in result['sources'])} were merged into {result['target']['albumtitle']}" + } + +@api.post("associate_albums_to_artist") +@authenticated_function(api=True) +@catch_exceptions +def associate_albums_to_artist(target_id,source_ids,remove=False): + result = database.associate_albums_to_artist(target_id,source_ids,remove=remove) + descword = "removed" if remove else "added" + if result: + return { + "status":"success", + "desc":f"{result['target']} was {descword} as album artist of {', '.join(src['albumtitle'] for src in result['sources'])}" + } + +@api.post("associate_tracks_to_artist") +@authenticated_function(api=True) +@catch_exceptions +def associate_tracks_to_artist(target_id,source_ids,remove=False): + result = database.associate_tracks_to_artist(target_id,source_ids,remove=remove) + descword = "removed" if remove else "added" + if result: + return { + "status":"success", + "desc":f"{result['target']} was {descword} as artist for {', '.join(src['title'] for src in result['sources'])}" + } + +@api.post("associate_tracks_to_album") +@authenticated_function(api=True) +@catch_exceptions +def associate_tracks_to_album(target_id,source_ids): + result = database.associate_tracks_to_album(target_id,source_ids) + if result: + return { + "status":"success", + "desc":f"{', '.join(src['title'] for src in result['sources'])} were " + f"added to {result['target']['albumtitle']}" if target_id else "removed from their album" + } + + @api.post("reparse_scrobble") @authenticated_function(api=True) @catch_exceptions diff --git a/maloja/cleanup.py b/maloja/cleanup.py index 9095114..62120b9 100644 --- a/maloja/cleanup.py +++ b/maloja/cleanup.py @@ -26,6 +26,7 @@ class CleanerAgent: self.rules_belongtogether = [r[1] for r in rawrules if r[0]=="belongtogether"] self.rules_notanartist = [r[1] for r in rawrules if r[0]=="notanartist"] self.rules_replacetitle = {r[1].lower():r[2] for r in rawrules if r[0]=="replacetitle"} + self.rules_replacealbumtitle = {r[1].lower():r[2] for r in rawrules if r[0]=="replacealbumtitle"} self.rules_replaceartist = {r[1].lower():r[2] for r in rawrules if r[0]=="replaceartist"} self.rules_ignoreartist = [r[1].lower() for r in rawrules if r[0]=="ignoreartist"] self.rules_addartists = {r[2].lower():(r[1].lower(),r[3]) for r in rawrules if r[0]=="addartists"} @@ -188,9 +189,14 @@ class CleanerAgent: if st in title.lower(): artists += self.rules_artistintitle[st].split("␟") return (title,artists) + def parseAlbumtitle(self,t): + if t.strip().lower() in self.rules_replacealbumtitle: + return self.rules_replacealbumtitle[t.strip().lower()] + t = t.replace("[","(").replace("]",")") - + t = t.strip() + return t def flatten(lis): diff --git a/maloja/data_files/cache/images/dummy b/maloja/data_files/cache/images/dummy new file mode 100644 index 0000000..e69de29 diff --git a/maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv b/maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv index a0c10b4..840bd84 100644 --- a/maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv +++ b/maloja/data_files/config/rules/predefined/krateng_kpopgirlgroups.tsv @@ -98,6 +98,7 @@ countas Jeongyeon TWICE countas Chaeyoung TWICE countas Nayeon TWICE countas Sana TWICE +countas MISAMO TWICE # AOA countas AOA Black AOA @@ -182,6 +183,9 @@ countas Akali K/DA # (G)I-DLE countas Soyeon (G)I-DLE countas Miyeon (G)I-DLE +countas Yuqi (G)I-DLE +countas Minnie (G)I-DLE +countas Shuhua (G)I-DLE replaceartist Jeon Soyeon Soyeon @@ -208,6 +212,12 @@ countas Wonyoung IVE countas Yujin IVE countas Gaeul IVE +# Pristin +countas Pristin V Pristin + +# CLC +countas Sorn CLC + # Popular Remixes artistintitle Areia Remix Areia artistintitle Areia Kpop Areia diff --git a/maloja/data_files/state/images/albums/dummy b/maloja/data_files/state/images/albums/dummy new file mode 100644 index 0000000..e69de29 diff --git a/maloja/database/__init__.py b/maloja/database/__init__.py index 15483d3..d35f456 100644 --- a/maloja/database/__init__.py +++ b/maloja/database/__init__.py @@ -1,17 +1,27 @@ # server from bottle import request, response, FormsDict + +# decorator that makes sure this function is only run in normal operation, +# not when we run a task that needs to access the database +def no_aux_mode(func): + def wrapper(*args,**kwargs): + from ..pkg_global import conf + if conf.AUX_MODE: return + return func(*args,**kwargs) + return wrapper + + # rest of the project from ..cleanup import CleanerAgent from .. import images -from ..malojatime import register_scrobbletime, time_stamps, ranges, alltime +from ..malojatime import register_scrobbletime, ranges, alltime, today, thisweek, thisyear, MTRangeComposite from ..malojauri import uri_to_internal, internal_to_uri, compose_querystring from ..thirdparty import proxy_scrobble_all from ..pkg_global.conf import data_dir, malojaconfig from ..apis import apikeystore #db from . import sqldb -from . import cached from . import dbcache from . import exceptions @@ -46,10 +56,15 @@ dbstatus = { + + + def waitfordb(func): def newfunc(*args,**kwargs): if not dbstatus['healthy']: raise exceptions.DatabaseNotBuilt() return func(*args,**kwargs) + + newfunc.__name__ = func.__name__ return newfunc @@ -93,12 +108,15 @@ def incoming_scrobble(rawscrobble,fix=True,client=None,api=None,dbconn=None): log(f"Incoming scrobble [Client: {client} | API: {api}]: {rawscrobble}") scrobbledict = rawscrobble_to_scrobbledict(rawscrobble, fix, client) + albumupdate = (malojaconfig["ALBUM_INFORMATION_TRUST"] == 'last') - sqldb.add_scrobble(scrobbledict,dbconn=dbconn) + + sqldb.add_scrobble(scrobbledict,update_album=albumupdate,dbconn=dbconn) proxy_scrobble_all(scrobbledict['track']['artists'],scrobbledict['track']['title'],scrobbledict['time']) dbcache.invalidate_caches(scrobbledict['time']) + #return {"status":"success","scrobble":scrobbledict} return scrobbledict @@ -130,8 +148,23 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None): scrobbleinfo = {**rawscrobble} if fix: scrobbleinfo['track_artists'],scrobbleinfo['track_title'] = cla.fullclean(scrobbleinfo['track_artists'],scrobbleinfo['track_title']) + if scrobbleinfo.get('album_artists'): + scrobbleinfo['album_artists'] = cla.parseArtists(scrobbleinfo['album_artists']) + if scrobbleinfo.get("album_title"): + scrobbleinfo['album_title'] = cla.parseAlbumtitle(scrobbleinfo['album_title']) scrobbleinfo['scrobble_time'] = scrobbleinfo.get('scrobble_time') or int(datetime.datetime.now(tz=datetime.timezone.utc).timestamp()) + # if we send [] as albumartists, it means various + # if we send nothing, the scrobbler just doesnt support it and we assume track artists + if ('album_title' in scrobbleinfo) and ('album_artists' not in scrobbleinfo): + scrobbleinfo['album_artists'] = scrobbleinfo.get('track_artists') + + # New plan, do this further down + # NONE always means there is simply no info, so make a guess or whatever the options say + # could use the track artists, but probably check if any album with the same name exists first + # various artists always needs to be specified via [] + # TODO + # processed info to internal scrobble dict scrobbledict = { "time":scrobbleinfo.get('scrobble_time'), @@ -139,7 +172,7 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None): "artists":scrobbleinfo.get('track_artists'), "title":scrobbleinfo.get('track_title'), "album":{ - "name":scrobbleinfo.get('album_name'), + "albumtitle":scrobbleinfo.get('album_title'), "artists":scrobbleinfo.get('album_artists') }, "length":scrobbleinfo.get('track_length') @@ -148,11 +181,15 @@ def rawscrobble_to_scrobbledict(rawscrobble, fix=True, client=None): "origin":f"client:{client}" if client else "generic", "extra":{ k:scrobbleinfo[k] for k in scrobbleinfo if k not in - ['scrobble_time','track_artists','track_title','track_length','scrobble_duration']#,'album_name','album_artists'] + ['scrobble_time','track_artists','track_title','track_length','scrobble_duration']#,'album_title','album_artists'] + # we still save album info in extra because the user might select majority album authority }, "rawscrobble":rawscrobble } + if not scrobbledict["track"]["album"]["albumtitle"]: + del scrobbledict["track"]["album"] + return scrobbledict @@ -184,12 +221,23 @@ def edit_track(id,trackinfo): return result +@waitfordb +def edit_album(id,albuminfo): + album = sqldb.get_album(id) + log(f"Renaming {album['albumtitle']} to {albuminfo['albumtitle']}") + result = sqldb.edit_album(id,albuminfo) + dbcache.invalidate_entity_cache() + dbcache.invalidate_caches() + + return result + @waitfordb def merge_artists(target_id,source_ids): sources = [sqldb.get_artist(id) for id in source_ids] target = sqldb.get_artist(target_id) log(f"Merging {sources} into {target}") - result = sqldb.merge_artists(target_id,source_ids) + sqldb.merge_artists(target_id,source_ids) + result = {'sources':sources,'target':target} dbcache.invalidate_entity_cache() dbcache.invalidate_caches() @@ -200,7 +248,20 @@ def merge_tracks(target_id,source_ids): sources = [sqldb.get_track(id) for id in source_ids] target = sqldb.get_track(target_id) log(f"Merging {sources} into {target}") - result = sqldb.merge_tracks(target_id,source_ids) + sqldb.merge_tracks(target_id,source_ids) + result = {'sources':sources,'target':target} + dbcache.invalidate_entity_cache() + dbcache.invalidate_caches() + + return result + +@waitfordb +def merge_albums(target_id,source_ids): + sources = [sqldb.get_album(id) for id in source_ids] + target = sqldb.get_album(target_id) + log(f"Merging {sources} into {target}") + sqldb.merge_albums(target_id,source_ids) + result = {'sources':sources,'target':target} dbcache.invalidate_entity_cache() dbcache.invalidate_caches() @@ -208,27 +269,95 @@ def merge_tracks(target_id,source_ids): +@waitfordb +def associate_albums_to_artist(target_id,source_ids,remove=False): + sources = [sqldb.get_album(id) for id in source_ids] + target = sqldb.get_artist(target_id) + if remove: + log(f"Removing {sources} from {target}") + sqldb.remove_artists_from_albums(artist_ids=[target_id],album_ids=source_ids) + else: + log(f"Adding {sources} into {target}") + sqldb.add_artists_to_albums(artist_ids=[target_id],album_ids=source_ids) + result = {'sources':sources,'target':target} + dbcache.invalidate_entity_cache() + dbcache.invalidate_caches() + + return result + +@waitfordb +def associate_tracks_to_artist(target_id,source_ids,remove=False): + sources = [sqldb.get_track(id) for id in source_ids] + target = sqldb.get_artist(target_id) + if remove: + log(f"Removing {sources} from {target}") + sqldb.remove_artists_from_tracks(artist_ids=[target_id],track_ids=source_ids) + else: + log(f"Adding {sources} into {target}") + sqldb.add_artists_to_tracks(artist_ids=[target_id],track_ids=source_ids) + result = {'sources':sources,'target':target} + dbcache.invalidate_entity_cache() + dbcache.invalidate_caches() + + return result + +@waitfordb +def associate_tracks_to_album(target_id,source_ids): + # target_id None means remove from current album! + sources = [sqldb.get_track(id) for id in source_ids] + if target_id: + target = sqldb.get_album(target_id) + log(f"Adding {sources} into {target}") + sqldb.add_tracks_to_albums({src:target_id for src in source_ids}) + else: + sqldb.remove_album(source_ids) + result = {'sources':sources,'target':target} + dbcache.invalidate_entity_cache() + dbcache.invalidate_caches() + + return result + + @waitfordb def get_scrobbles(dbconn=None,**keys): (since,to) = keys.get('timerange').timestamps() - if 'artist' in keys: - result = sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,dbconn=dbconn) - elif 'track' in keys: - result = sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to,dbconn=dbconn) + + reverse = keys.get('reverse',True) # comaptibility with old calls + if keys.get('perpage',math.inf) is not math.inf: + limit = (keys.get('page',0)+1) * keys.get('perpage',100) + behead = keys.get('page',0) * keys.get('perpage',100) else: - result = sqldb.get_scrobbles(since=since,to=to,dbconn=dbconn) + limit = None + behead = 0 + + + associated = keys.get('associated',False) + if 'artist' in keys: + result = sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,associated=associated,limit=limit,reverse=reverse,dbconn=dbconn) + elif 'track' in keys: + result = sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to,limit=limit,reverse=reverse,dbconn=dbconn) + elif 'album' in keys: + result = sqldb.get_scrobbles_of_album(album=keys['album'],since=since,to=to,limit=limit,reverse=reverse,dbconn=dbconn) + else: + result = sqldb.get_scrobbles(since=since,to=to,limit=limit,reverse=reverse,dbconn=dbconn) #return result[keys['page']*keys['perpage']:(keys['page']+1)*keys['perpage']] - return list(reversed(result)) + + #print(result) + + return list(result[behead:]) @waitfordb def get_scrobbles_num(dbconn=None,**keys): (since,to) = keys.get('timerange').timestamps() + associated = keys.get('associated',False) if 'artist' in keys: - result = len(sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,resolve_references=False,dbconn=dbconn)) + result = len(sqldb.get_scrobbles_of_artist(artist=keys['artist'],since=since,to=to,associated=associated,resolve_references=False,dbconn=dbconn)) elif 'track' in keys: result = len(sqldb.get_scrobbles_of_track(track=keys['track'],since=since,to=to,resolve_references=False,dbconn=dbconn)) + elif 'album' in keys: + result = len(sqldb.get_scrobbles_of_album(album=keys['album'],since=since,to=to,resolve_references=False,dbconn=dbconn)) else: result = sqldb.get_scrobbles_num(since=since,to=to,dbconn=dbconn) return result @@ -244,59 +373,184 @@ def get_tracks(dbconn=None,**keys): return result @waitfordb -def get_artists(dbconn=None): - return sqldb.get_artists(dbconn=dbconn) - - -@waitfordb -def get_charts_artists(dbconn=None,**keys): - (since,to) = keys.get('timerange').timestamps() - result = sqldb.count_scrobbles_by_artist(since=since,to=to,dbconn=dbconn) +def get_albums(dbconn=None,**keys): + if keys.get('artist') is None: + result = sqldb.get_albums(dbconn=dbconn) + else: + result = sqldb.get_albums_of_artists([sqldb.get_artist_id(keys.get('artist'),create_new=False)],dbconn=dbconn) return result @waitfordb -def get_charts_tracks(dbconn=None,**keys): +def get_artists(dbconn=None): + return sqldb.get_artists(dbconn=dbconn) + +@waitfordb +def get_albums_artist_appears_on(dbconn=None,**keys): + + artist_id = sqldb.get_artist_id(keys['artist'],dbconn=dbconn) + + albums = sqldb.get_albums_artists_appear_on([artist_id],dbconn=dbconn).get(artist_id) or [] + ownalbums = sqldb.get_albums_of_artists([artist_id],dbconn=dbconn).get(artist_id) or [] + + result = { + "own_albums":ownalbums, + "appears_on":[a for a in albums if a not in ownalbums] + } + + return result + +@waitfordb +def get_tracks_without_album(dbconn=None,resolve_ids=True): + return get_charts_tracks(album=None,timerange=alltime(),resolve_ids=resolve_ids,dbconn=dbconn) + +@waitfordb +def get_charts_artists(dbconn=None,resolve_ids=True,**keys): + (since,to) = keys.get('timerange').timestamps() + separate = keys.get('separate',False) + result = sqldb.count_scrobbles_by_artist(since=since,to=to,resolve_ids=resolve_ids,associated=(not separate),dbconn=dbconn) + + if resolve_ids: + # only add associated info if we resolve + map = sqldb.get_associated_artist_map(artist_ids=[entry['artist_id'] for entry in result if 'artist_id' in entry]) + for entry in result: + if "artist_id" in entry: + entry['associated_artists'] = map[entry['artist_id']] + return result + +@waitfordb +def get_charts_tracks(dbconn=None,resolve_ids=True,**keys): (since,to) = keys.get('timerange').timestamps() if 'artist' in keys: - result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],dbconn=dbconn) + result = sqldb.count_scrobbles_by_track_of_artist(since=since,to=to,artist=keys['artist'],associated=keys.get('associated',False),resolve_ids=resolve_ids,dbconn=dbconn) + elif 'album' in keys: + result = sqldb.count_scrobbles_by_track_of_album(since=since,to=to,album=keys['album'],resolve_ids=resolve_ids,dbconn=dbconn) else: - result = sqldb.count_scrobbles_by_track(since=since,to=to,dbconn=dbconn) + result = sqldb.count_scrobbles_by_track(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn) + return result + +@waitfordb +def get_charts_albums(dbconn=None,resolve_ids=True,only_own_albums=False,**keys): + # TODO: different scrobble numbers for only own tracks on own album etc? + (since,to) = keys.get('timerange').timestamps() + + if 'artist' in keys: + result = sqldb.count_scrobbles_by_album_combined(since=since,to=to,artist=keys['artist'],associated=keys.get('associated',False),resolve_ids=resolve_ids,dbconn=dbconn) + if only_own_albums: + # TODO: this doesnt take associated into account and doesnt change ranks + result = [e for e in result if keys['artist'] in (e['album']['artists'] or [])] + else: + result = sqldb.count_scrobbles_by_album(since=since,to=to,resolve_ids=resolve_ids,dbconn=dbconn) return result @waitfordb def get_pulse(dbconn=None,**keys): + # amountkeys for pulse and performance aren't really necessary + # since the amount of entries is completely determined by the time keys + # but lets just include it in case + reverse = keys.get('reverse',False) + if keys.get('perpage',math.inf) is not math.inf: + limit = (keys.get('page',0)+1) * keys.get('perpage',100) + behead = keys.get('page',0) * keys.get('perpage',100) + else: + limit = math.inf + behead = 0 + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) + if reverse: rngs = reversed(list(rngs)) results = [] for rng in rngs: + + # count down how many we need + if limit==0: + break + limit -= 1 + + # skip prev pages + if behead>0: + behead -= 1 + continue + res = get_scrobbles_num(timerange=rng,**{k:keys[k] for k in keys if k != 'timerange'},dbconn=dbconn) - results.append({"range":rng,"scrobbles":res}) + if keys.get('artist') and keys.get('associated',False): + res_real = get_scrobbles_num(timerange=rng,**{k:keys[k] for k in keys if k not in ['timerange','associated']},associated=False,dbconn=dbconn) + # this isnt really efficient, we could do that in one db call, but i dont wanna reorganize rn + else: + res_real = res + results.append({"range":rng,"scrobbles":res,"real_scrobbles":res_real}) return results @waitfordb def get_performance(dbconn=None,**keys): + # amountkeys for pulse and performance aren't really necessary + # since the amount of entries is completely determined by the time keys + # but lets just include it in case + reverse = keys.get('reverse',False) + if keys.get('perpage',math.inf) is not math.inf: + limit = (keys.get('page',0)+1) * keys.get('perpage',100) + behead = keys.get('page',0) * keys.get('perpage',100) + else: + limit = math.inf + behead = 0 + + separate = keys.get('separate') + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) + if reverse: rngs = reversed(list(rngs)) results = [] for rng in rngs: + + # count down how many we need + if limit==0: + break + limit -= 1 + + # skip prev pages + if behead>0: + behead -= 1 + continue + + + + + + if "track" in keys: - track = sqldb.get_track(sqldb.get_track_id(keys['track'],dbconn=dbconn),dbconn=dbconn) - charts = get_charts_tracks(timerange=rng,dbconn=dbconn) + track_id = sqldb.get_track_id(keys['track'],create_new=False,dbconn=dbconn) + if not track_id: + raise exceptions.TrackDoesNotExist(keys['track']) + #track = sqldb.get_track(track_id,dbconn=dbconn) + charts = get_charts_tracks(timerange=rng,resolve_ids=False,dbconn=dbconn) rank = None for c in charts: - if c["track"] == track: + if c["track_id"] == track_id: rank = c["rank"] break elif "artist" in keys: - artist = sqldb.get_artist(sqldb.get_artist_id(keys['artist'],dbconn=dbconn),dbconn=dbconn) + artist_id = sqldb.get_artist_id(keys['artist'],create_new=False,dbconn=dbconn) + if not artist_id: + raise exceptions.ArtistDoesNotExist(keys['artist']) + #artist = sqldb.get_artist(artist_id,dbconn=dbconn) # ^this is the most useless line in programming history # but I like consistency - charts = get_charts_artists(timerange=rng,dbconn=dbconn) + charts = get_charts_artists(timerange=rng,resolve_ids=False,separate=separate,dbconn=dbconn) rank = None for c in charts: - if c["artist"] == artist: + if c["artist_id"] == artist_id: + rank = c["rank"] + break + elif "album" in keys: + album_id = sqldb.get_album_id(keys['album'],create_new=False,dbconn=dbconn) + if not album_id: + raise exceptions.AlbumDoesNotExist(keys['album']) + #album = sqldb.get_album(album_id,dbconn=dbconn) + charts = get_charts_albums(timerange=rng,resolve_ids=False,dbconn=dbconn) + rank = None + for c in charts: + if c["album_id"] == album_id: rank = c["rank"] break else: @@ -308,15 +562,17 @@ def get_performance(dbconn=None,**keys): @waitfordb def get_top_artists(dbconn=None,**keys): + separate = keys.get('separate') + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) results = [] for rng in rngs: try: - res = get_charts_artists(timerange=rng,dbconn=dbconn)[0] - results.append({"range":rng,"artist":res["artist"],"scrobbles":res["scrobbles"]}) + res = get_charts_artists(timerange=rng,separate=separate,dbconn=dbconn)[0] + results.append({"range":rng,"artist":res["artist"],"scrobbles":res["scrobbles"],"real_scrobbles":res["real_scrobbles"],"associated_artists":sqldb.get_associated_artists(res["artist"])}) except Exception: - results.append({"range":rng,"artist":None,"scrobbles":0}) + results.append({"range":rng,"artist":None,"scrobbles":0,"real_scrobbles":0}) return results @@ -336,48 +592,112 @@ def get_top_tracks(dbconn=None,**keys): return results +@waitfordb +def get_top_albums(dbconn=None,**keys): + + rngs = ranges(**{k:keys[k] for k in keys if k in ["since","to","within","timerange","step","stepn","trail"]}) + results = [] + + for rng in rngs: + try: + res = get_charts_albums(timerange=rng,dbconn=dbconn)[0] + results.append({"range":rng,"album":res["album"],"scrobbles":res["scrobbles"]}) + except Exception: + results.append({"range":rng,"album":None,"scrobbles":0}) + + return results + @waitfordb def artist_info(dbconn=None,**keys): artist = keys.get('artist') if artist is None: raise exceptions.MissingEntityParameter() - artist_id = sqldb.get_artist_id(artist,dbconn=dbconn) + artist_id = sqldb.get_artist_id(artist,create_new=False,dbconn=dbconn) + if not artist_id: raise exceptions.ArtistDoesNotExist(artist) + artist = sqldb.get_artist(artist_id,dbconn=dbconn) alltimecharts = get_charts_artists(timerange=alltime(),dbconn=dbconn) - scrobbles = get_scrobbles_num(artist=artist,timerange=alltime(),dbconn=dbconn) #we cant take the scrobble number from the charts because that includes all countas scrobbles - try: - c = [e for e in alltimecharts if e["artist"] == artist][0] + scrobbles = get_scrobbles_num(artist=artist,timerange=alltime(),dbconn=dbconn) + albums = sqldb.get_albums_of_artists(set([artist_id]),dbconn=dbconn) + isalbumartist = len(albums.get(artist_id,[]))>0 + + cert = None + own_track_charts = get_charts_tracks(timerange=alltime(),resolve_ids=False,artist=artist,dbconn=dbconn) + own_album_charts = get_charts_albums(timerange=alltime(),resolve_ids=True,artist=artist,dbconn=dbconn) + # we resolve ids here which we don't need to. however, on the jinja page we make that same call + # later again with resolve ids, so its a cache miss and it doubles page load time + # TODO: find better solution + if own_track_charts: + c = own_track_charts[0] + tscrobbles = c["scrobbles"] + threshold_gold, threshold_platinum, threshold_diamond = malojaconfig["SCROBBLES_GOLD","SCROBBLES_PLATINUM","SCROBBLES_DIAMOND"] + if tscrobbles >= threshold_diamond: cert = "diamond" + elif tscrobbles >= threshold_platinum: cert = "platinum" + elif tscrobbles >= threshold_gold: cert = "gold" + if own_album_charts: + c = own_album_charts[0] + ascrobbles = c["scrobbles"] + threshold_gold, threshold_platinum, threshold_diamond = malojaconfig["SCROBBLES_GOLD_ALBUM","SCROBBLES_PLATINUM_ALBUM","SCROBBLES_DIAMOND_ALBUM"] + if ascrobbles >= threshold_diamond: cert = "diamond" + elif ascrobbles >= threshold_platinum and cert != "diamond": cert = "platinum" + elif ascrobbles >= threshold_gold and not cert: cert = "gold" + + twk = thisweek() + tyr = thisyear() + + # base info for everyone + result = { + "artist":artist, + "scrobbles":scrobbles, + "id":artist_id, + "isalbumartist":isalbumartist, + "certification":cert, + } + + # check if credited to someone else + parent_artists = sqldb.get_credited_artists(artist) + if len(parent_artists) == 0: + c = [e for e in alltimecharts if e["artist"] == artist] + position = c[0]["rank"] if len(c) > 0 else None others = sqldb.get_associated_artists(artist,dbconn=dbconn) - position = c["rank"] - return { - "artist":artist, - "scrobbles":scrobbles, + result.update({ "position":position, "associated":others, "medals":{ - "gold": [year for year in cached.medals_artists if artist_id in cached.medals_artists[year]['gold']], - "silver": [year for year in cached.medals_artists if artist_id in cached.medals_artists[year]['silver']], - "bronze": [year for year in cached.medals_artists if artist_id in cached.medals_artists[year]['bronze']], + "gold": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('artist_id') == artist_id) and (e.get('rank') == 1) for e in + sqldb.count_scrobbles_by_artist(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )], + "silver": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('artist_id') == artist_id) and (e.get('rank') == 2) for e in + sqldb.count_scrobbles_by_artist(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )], + "bronze": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('artist_id') == artist_id) and (e.get('rank') == 3) for e in + sqldb.count_scrobbles_by_artist(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )] }, - "topweeks":len([e for e in cached.weekly_topartists if e == artist_id]), - "id":artist_id - } - except Exception: - # if the artist isnt in the charts, they are not being credited and we - # need to show information about the credited one - replaceartist = sqldb.get_credited_artists(artist)[0] + "topweeks":len([ + week for week in ranges(step="week") if (week != twk) and any( + (e.get('artist_id') == artist_id) and (e.get('rank') == 1) for e in + sqldb.count_scrobbles_by_artist(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,associated=True,dbconn=dbconn) + ) + # we don't need to check the whole thing, just until rank is lower, but... well, its a list comprehension + ]) + }) + + else: + replaceartist = parent_artists[0] c = [e for e in alltimecharts if e["artist"] == replaceartist][0] position = c["rank"] - return { - "artist":artist, + result.update({ "replace":replaceartist, - "scrobbles":scrobbles, - "position":position, - "id":artist_id - } + "position":position + }) + return result @@ -387,12 +707,14 @@ def track_info(dbconn=None,**keys): track = keys.get('track') if track is None: raise exceptions.MissingEntityParameter() - track_id = sqldb.get_track_id(track,dbconn=dbconn) + track_id = sqldb.get_track_id(track,create_new=False,dbconn=dbconn) + if not track_id: raise exceptions.TrackDoesNotExist(track) + track = sqldb.get_track(track_id,dbconn=dbconn) - alltimecharts = get_charts_tracks(timerange=alltime(),dbconn=dbconn) + alltimecharts = get_charts_tracks(timerange=alltime(),resolve_ids=False,dbconn=dbconn) #scrobbles = get_scrobbles_num(track=track,timerange=alltime()) - c = [e for e in alltimecharts if e["track"] == track][0] + c = [e for e in alltimecharts if e["track_id"] == track_id][0] scrobbles = c["scrobbles"] position = c["rank"] cert = None @@ -401,22 +723,129 @@ def track_info(dbconn=None,**keys): elif scrobbles >= threshold_platinum: cert = "platinum" elif scrobbles >= threshold_gold: cert = "gold" + twk = thisweek() + tyr = thisyear() return { "track":track, "scrobbles":scrobbles, "position":position, "medals":{ - "gold": [year for year in cached.medals_tracks if track_id in cached.medals_tracks[year]['gold']], - "silver": [year for year in cached.medals_tracks if track_id in cached.medals_tracks[year]['silver']], - "bronze": [year for year in cached.medals_tracks if track_id in cached.medals_tracks[year]['bronze']], + "gold": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('track_id') == track_id) and (e.get('rank') == 1) for e in + sqldb.count_scrobbles_by_track(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )], + "silver": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('track_id') == track_id) and (e.get('rank') == 2) for e in + sqldb.count_scrobbles_by_track(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )], + "bronze": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('track_id') == track_id) and (e.get('rank') == 3) for e in + sqldb.count_scrobbles_by_track(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )] }, "certification":cert, - "topweeks":len([e for e in cached.weekly_toptracks if e == track_id]), + "topweeks":len([ + week for week in ranges(step="week") if (week != twk) and any( + (e.get('track_id') == track_id) and (e.get('rank') == 1) for e in + sqldb.count_scrobbles_by_track(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=dbconn) + ) + ]), "id":track_id } +@waitfordb +def album_info(dbconn=None,reduced=False,**keys): + + album = keys.get('album') + if album is None: raise exceptions.MissingEntityParameter() + + album_id = sqldb.get_album_id(album,create_new=False,dbconn=dbconn) + if not album_id: raise exceptions.AlbumDoesNotExist(album) + + album = sqldb.get_album(album_id,dbconn=dbconn) + + extrainfo = {} + + if reduced: + scrobbles = get_scrobbles_num(album=album,timerange=alltime()) + else: + alltimecharts = get_charts_albums(timerange=alltime(),dbconn=dbconn) + c = [e for e in alltimecharts if e["album"] == album][0] + scrobbles = c["scrobbles"] + position = c["rank"] + extrainfo['position'] = position + + cert = None + threshold_gold, threshold_platinum, threshold_diamond = malojaconfig["SCROBBLES_GOLD_ALBUM","SCROBBLES_PLATINUM_ALBUM","SCROBBLES_DIAMOND_ALBUM"] + if scrobbles >= threshold_diamond: cert = "diamond" + elif scrobbles >= threshold_platinum: cert = "platinum" + elif scrobbles >= threshold_gold: cert = "gold" + + if reduced: + pass + else: + twk = thisweek() + tyr = thisyear() + extrainfo.update({ + "medals":{ + "gold": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('album_id') == album_id) and (e.get('rank') == 1) for e in + sqldb.count_scrobbles_by_album(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )], + "silver": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('album_id') == album_id) and (e.get('rank') == 2) for e in + sqldb.count_scrobbles_by_album(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )], + "bronze": [year.desc() for year in ranges(step='year') if (year != tyr) and any( + (e.get('album_id') == album_id) and (e.get('rank') == 3) for e in + sqldb.count_scrobbles_by_album(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=dbconn) + )] + }, + "topweeks":len([ + week for week in ranges(step="week") if (week != twk) and any( + (e.get('album_id') == album_id) and (e.get('rank') == 1) for e in + sqldb.count_scrobbles_by_album(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=dbconn) + ) + ]) + }) + + return { + "album":album, + "scrobbles":scrobbles, + "certification":cert, + "id":album_id, + **extrainfo + } + + + +### TODO: FIND COOL ALGORITHM TO SELECT FEATURED STUFF +@waitfordb +def get_featured(dbconn=None): + # temporary stand-in + ranges = [ + MTRangeComposite(since=today().next(-14),to=today()), + MTRangeComposite(since=thisweek().next(-12),to=thisweek()), + MTRangeComposite(since=thisweek().next(-52),to=thisweek()), + alltime() + ] + funcs = { + "artist": (get_charts_artists,{'associated':False}), + "album": (get_charts_albums,{}), + "track": (get_charts_tracks,{}) + } + result = {t:None for t in funcs} + + for entity_type in funcs: + for r in ranges: + func,kwargs = funcs[entity_type] + chart = func(timerange=r,**kwargs) + if chart: + result[entity_type] = chart[0][entity_type] + break + return result def get_predefined_rulesets(dbconn=None): validchars = "-_abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" @@ -464,6 +893,7 @@ def start_db(): # Upgrade database from .. import upgrade upgrade.upgrade_db(sqldb.add_scrobbles) + upgrade.parse_old_albums() # Load temporary tables from . import associated @@ -478,14 +908,15 @@ def start_db(): except IndexError: register_scrobbletime(int(datetime.datetime.now().timestamp())) - - # create cached information - cached.update_medals() - cached.update_weekly() - dbstatus['complete'] = True - + # cache some stuff that we'll probably need + with sqldb.engine.connect() as dbconn: + with dbconn.begin(): + for week in ranges(step='week'): + sqldb.count_scrobbles_by_artist(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,associated=True,dbconn=dbconn) + sqldb.count_scrobbles_by_track(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=dbconn) + sqldb.count_scrobbles_by_album(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=dbconn) @@ -497,4 +928,7 @@ def db_search(query,type=None): results = sqldb.search_artist(query) if type=="TRACK": results = sqldb.search_track(query) + if type=="ALBUM": + results = sqldb.search_album(query) + return results diff --git a/maloja/database/cached.py b/maloja/database/cached.py deleted file mode 100644 index ea39a29..0000000 --- a/maloja/database/cached.py +++ /dev/null @@ -1,74 +0,0 @@ -# for information that is not authorative, but should be saved anyway because it -# changes infrequently and DB access is expensive - -from doreah.regular import runyearly, rundaily -from .. import database -from . import sqldb -from .. import malojatime as mjt - - - -medals_artists = { - # year: {'gold':[],'silver':[],'bronze':[]} -} -medals_tracks = { - # year: {'gold':[],'silver':[],'bronze':[]} -} - -weekly_topartists = [] -weekly_toptracks = [] - -@runyearly -def update_medals(): - - global medals_artists, medals_tracks - medals_artists.clear() - medals_tracks.clear() - - with sqldb.engine.begin() as conn: - for year in mjt.ranges(step="year"): - if year == mjt.thisyear(): break - - charts_artists = sqldb.count_scrobbles_by_artist(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=conn) - charts_tracks = sqldb.count_scrobbles_by_track(since=year.first_stamp(),to=year.last_stamp(),resolve_ids=False,dbconn=conn) - - entry_artists = {'gold':[],'silver':[],'bronze':[]} - entry_tracks = {'gold':[],'silver':[],'bronze':[]} - medals_artists[year.desc()] = entry_artists - medals_tracks[year.desc()] = entry_tracks - - for entry in charts_artists: - if entry['rank'] == 1: entry_artists['gold'].append(entry['artist_id']) - elif entry['rank'] == 2: entry_artists['silver'].append(entry['artist_id']) - elif entry['rank'] == 3: entry_artists['bronze'].append(entry['artist_id']) - else: break - for entry in charts_tracks: - if entry['rank'] == 1: entry_tracks['gold'].append(entry['track_id']) - elif entry['rank'] == 2: entry_tracks['silver'].append(entry['track_id']) - elif entry['rank'] == 3: entry_tracks['bronze'].append(entry['track_id']) - else: break - - - - -@rundaily -def update_weekly(): - - global weekly_topartists, weekly_toptracks - weekly_topartists.clear() - weekly_toptracks.clear() - - with sqldb.engine.begin() as conn: - for week in mjt.ranges(step="week"): - if week == mjt.thisweek(): break - - - charts_artists = sqldb.count_scrobbles_by_artist(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=conn) - charts_tracks = sqldb.count_scrobbles_by_track(since=week.first_stamp(),to=week.last_stamp(),resolve_ids=False,dbconn=conn) - - for entry in charts_artists: - if entry['rank'] == 1: weekly_topartists.append(entry['artist_id']) - else: break - for entry in charts_tracks: - if entry['rank'] == 1: weekly_toptracks.append(entry['track_id']) - else: break diff --git a/maloja/database/dbcache.py b/maloja/database/dbcache.py index 9b092a9..b82f2c8 100644 --- a/maloja/database/dbcache.py +++ b/maloja/database/dbcache.py @@ -10,7 +10,7 @@ from doreah.regular import runhourly from doreah.logging import log from ..pkg_global.conf import malojaconfig - +from . import no_aux_mode if malojaconfig['USE_GLOBAL_CACHE']: @@ -21,6 +21,7 @@ if malojaconfig['USE_GLOBAL_CACHE']: @runhourly + @no_aux_mode def maintenance(): print_stats() trim_cache() @@ -42,6 +43,7 @@ if malojaconfig['USE_GLOBAL_CACHE']: conn = None global hits, misses key = (serialize(args),serialize(kwargs), inner_func, kwargs.get("since"), kwargs.get("to")) + # TODO: also factor in default values to get better chance of hits try: return cache[key] @@ -50,6 +52,8 @@ if malojaconfig['USE_GLOBAL_CACHE']: cache[key] = result return result + outer_func.__name__ = f"CACHD_{inner_func.__name__}" + return outer_func @@ -80,22 +84,23 @@ if malojaconfig['USE_GLOBAL_CACHE']: return outer_func + @no_aux_mode def invalidate_caches(scrobbletime=None): + cleared, kept = 0, 0 for k in cache.keys(): # VERY BIG TODO: differentiate between None as in 'unlimited timerange' and None as in 'time doesnt matter here'! - if scrobbletime is None or (k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4]): + if scrobbletime is None or ((k[3] is None or scrobbletime >= k[3]) and (k[4] is None or scrobbletime <= k[4])): cleared += 1 del cache[k] else: kept += 1 log(f"Invalidated {cleared} of {cleared+kept} DB cache entries") - + @no_aux_mode def invalidate_entity_cache(): entitycache.clear() - def trim_cache(): ramprct = psutil.virtual_memory().percent if ramprct > malojaconfig["DB_MAX_MEMORY"]: @@ -132,7 +137,7 @@ else: def serialize(obj): try: return serialize(obj.hashable()) - except Exception: + except AttributeError: try: return json.dumps(obj) except Exception: diff --git a/maloja/database/exceptions.py b/maloja/database/exceptions.py index a2dadd9..534dc1e 100644 --- a/maloja/database/exceptions.py +++ b/maloja/database/exceptions.py @@ -11,12 +11,14 @@ class TrackExists(EntityExists): class ArtistExists(EntityExists): pass +class AlbumExists(EntityExists): + pass class DatabaseNotBuilt(HTTPError): def __init__(self): super().__init__( status=503, - body="The Maloja Database is being upgraded to Version 3. This could take quite a long time! (~ 2-5 minutes per 10 000 scrobbles)", + body="The Maloja Database is being upgraded to support new Maloja features. This could take a while.", headers={"Retry-After":120} ) @@ -27,3 +29,19 @@ class MissingScrobbleParameters(Exception): class MissingEntityParameter(Exception): pass + +class EntityDoesNotExist(HTTPError): + entitytype = 'Entity' + def __init__(self,entitydict): + self.entitydict = entitydict + super().__init__( + status=404, + body=f"The {self.entitytype} '{self.entitydict}' does not exist in the database." + ) + +class ArtistDoesNotExist(EntityDoesNotExist): + entitytype = 'Artist' +class AlbumDoesNotExist(EntityDoesNotExist): + entitytype = 'Album' +class TrackDoesNotExist(EntityDoesNotExist): + entitytype = 'Track' diff --git a/maloja/database/sqldb.py b/maloja/database/sqldb.py index 61f0b38..0748170 100644 --- a/maloja/database/sqldb.py +++ b/maloja/database/sqldb.py @@ -1,4 +1,5 @@ import sqlalchemy as sql +from sqlalchemy.dialects.sqlite import insert as sqliteinsert import json import unicodedata import math @@ -6,8 +7,9 @@ from datetime import datetime from threading import Lock from ..pkg_global.conf import data_dir -from .dbcache import cached_wrapper, cached_wrapper_individual +from .dbcache import cached_wrapper, cached_wrapper_individual, invalidate_caches, invalidate_entity_cache from . import exceptions as exc +from . import no_aux_mode from doreah.logging import log from doreah.regular import runhourly, runmonthly @@ -19,46 +21,81 @@ from doreah.regular import runhourly, runmonthly DBTABLES = { # name - type - foreign key - kwargs + '_maloja':{ + 'columns':[ + ("key", sql.String, {'primary_key':True}), + ("value", sql.String, {}) + ], + 'extraargs':(),'extrakwargs':{} + }, 'scrobbles':{ 'columns':[ - ("timestamp", sql.Integer, {'primary_key':True}), - ("rawscrobble", sql.String, {}), - ("origin", sql.String, {}), - ("duration", sql.Integer, {}), - ("track_id", sql.Integer, sql.ForeignKey('tracks.id'), {}), - ("extra", sql.String, {}) + ("timestamp", sql.Integer, {'primary_key':True}), + ("rawscrobble", sql.String, {}), + ("origin", sql.String, {}), + ("duration", sql.Integer, {}), + ("track_id", sql.Integer, sql.ForeignKey('tracks.id'), {}), + ("extra", sql.String, {}) ], 'extraargs':(),'extrakwargs':{} }, 'tracks':{ 'columns':[ - ("id", sql.Integer, {'primary_key':True}), - ("title", sql.String, {}), - ("title_normalized",sql.String, {}), - ("length", sql.Integer, {}) + ("id", sql.Integer, {'primary_key':True}), + ("title", sql.String, {}), + ("title_normalized", sql.String, {}), + ("length", sql.Integer, {}), + ("album_id", sql.Integer, sql.ForeignKey('albums.id'), {}) ], 'extraargs':(),'extrakwargs':{'sqlite_autoincrement':True} }, 'artists':{ 'columns':[ - ("id", sql.Integer, {'primary_key':True}), - ("name", sql.String, {}), - ("name_normalized", sql.String, {}) + ("id", sql.Integer, {'primary_key':True}), + ("name", sql.String, {}), + ("name_normalized", sql.String, {}) + ], + 'extraargs':(),'extrakwargs':{'sqlite_autoincrement':True} + }, + 'albums':{ + 'columns':[ + ("id", sql.Integer, {'primary_key':True}), + ("albtitle", sql.String, {}), + ("albtitle_normalized", sql.String, {}) + #("albumartist", sql.String, {}) + # when an album has no artists, always use 'Various Artists' ], 'extraargs':(),'extrakwargs':{'sqlite_autoincrement':True} }, 'trackartists':{ 'columns':[ - ("id", sql.Integer, {'primary_key':True}), - ("artist_id", sql.Integer, sql.ForeignKey('artists.id'), {}), - ("track_id", sql.Integer, sql.ForeignKey('tracks.id'), {}) + ("id", sql.Integer, {'primary_key':True}), + ("artist_id", sql.Integer, sql.ForeignKey('artists.id'), {}), + ("track_id", sql.Integer, sql.ForeignKey('tracks.id'), {}) ], 'extraargs':(sql.UniqueConstraint('artist_id', 'track_id'),),'extrakwargs':{} }, + 'albumartists':{ + 'columns':[ + ("id", sql.Integer, {'primary_key':True}), + ("artist_id", sql.Integer, sql.ForeignKey('artists.id'), {}), + ("album_id", sql.Integer, sql.ForeignKey('albums.id'), {}) + ], + 'extraargs':(sql.UniqueConstraint('artist_id', 'album_id'),),'extrakwargs':{} + }, +# 'albumtracks':{ +# # tracks can be in multiple albums +# 'columns':[ +# ("id", sql.Integer, {'primary_key':True}), +# ("album_id", sql.Integer, sql.ForeignKey('albums.id'), {}), +# ("track_id", sql.Integer, sql.ForeignKey('tracks.id'), {}) +# ], +# 'extraargs':(sql.UniqueConstraint('album_id', 'track_id'),),'extrakwargs':{} +# }, 'associated_artists':{ 'columns':[ - ("source_artist", sql.Integer, sql.ForeignKey('artists.id'), {}), - ("target_artist", sql.Integer, sql.ForeignKey('artists.id'), {}) + ("source_artist", sql.Integer, sql.ForeignKey('artists.id'), {}), + ("target_artist", sql.Integer, sql.ForeignKey('artists.id'), {}) ], 'extraargs':(sql.UniqueConstraint('source_artist', 'target_artist'),),'extrakwargs':{} } @@ -120,8 +157,32 @@ def connection_provider(func): return func(*args,**kwargs) wrapper.__innerfunc__ = func + wrapper.__name__ = f"CONPR_{func.__name__}" return wrapper +@connection_provider +def get_maloja_info(keys,dbconn=None): + op = DB['_maloja'].select().where( + DB['_maloja'].c.key.in_(keys) + ) + result = dbconn.execute(op).all() + + info = {} + for row in result: + info[row.key] = row.value + return info + +@connection_provider +def set_maloja_info(info,dbconn=None): + for k in info: + op = sqliteinsert(DB['_maloja']).values( + key=k, value=info[k] + ).on_conflict_do_update( + index_elements=['key'], + set_={'value':info[k]} + ) + dbconn.execute(op) + ##### DB <-> Dict translations ## ATTENTION ALL ADVENTURERS @@ -138,7 +199,7 @@ def connection_provider(func): # "artists":list, # "title":string, # "album":{ -# "name":string, +# "albumtitle":string, # "artists":list # }, # "length":None @@ -185,11 +246,12 @@ def scrobble_db_to_dict(row,dbconn=None): def tracks_db_to_dict(rows,dbconn=None): artists = get_artists_of_tracks(set(row.id for row in rows),dbconn=dbconn) + albums = get_albums_map(set(row.album_id for row in rows),dbconn=dbconn) return [ { "artists":artists[row.id], "title":row.title, - #"album": + "album":albums.get(row.album_id), "length":row.length } for row in rows @@ -207,18 +269,31 @@ def artists_db_to_dict(rows,dbconn=None): def artist_db_to_dict(row,dbconn=None): return artists_db_to_dict([row],dbconn=dbconn)[0] +def albums_db_to_dict(rows,dbconn=None): + artists = get_artists_of_albums(set(row.id for row in rows),dbconn=dbconn) + return [ + { + "artists":artists.get(row.id), + "albumtitle":row.albtitle, + } + for row in rows + ] + +def album_db_to_dict(row,dbconn=None): + return albums_db_to_dict([row],dbconn=dbconn)[0] + ### DICT -> DB # These should return None when no data is in the dict so they can be used for update statements -def scrobble_dict_to_db(info,dbconn=None): +def scrobble_dict_to_db(info,update_album=False,dbconn=None): return { "timestamp":info.get('time'), "origin":info.get('origin'), "duration":info.get('duration'), - "track_id":get_track_id(info.get('track'),dbconn=dbconn), + "track_id":get_track_id(info.get('track'),update_album=update_album,dbconn=dbconn), "extra":json.dumps(info.get('extra')) if info.get('extra') else None, "rawscrobble":json.dumps(info.get('rawscrobble')) if info.get('rawscrobble') else None } @@ -236,25 +311,33 @@ def artist_dict_to_db(info,dbconn=None): "name_normalized":normalize_name(info) } +def album_dict_to_db(info,dbconn=None): + return { + "albtitle":info.get('albumtitle'), + "albtitle_normalized":normalize_name(info.get('albumtitle')) + } + ##### Actual Database interactions +# TODO: remove all resolve_id args and do that logic outside the caching to improve hit chances +# TODO: maybe also factor out all intitial get entity funcs (some here, some in __init__) and throw exceptions @connection_provider -def add_scrobble(scrobbledict,dbconn=None): - add_scrobbles([scrobbledict],dbconn=dbconn) +def add_scrobble(scrobbledict,update_album=False,dbconn=None): + add_scrobbles([scrobbledict],update_album=update_album,dbconn=dbconn) @connection_provider -def add_scrobbles(scrobbleslist,dbconn=None): +def add_scrobbles(scrobbleslist,update_album=False,dbconn=None): with SCROBBLE_LOCK: ops = [ DB['scrobbles'].insert().values( - **scrobble_dict_to_db(s,dbconn=dbconn) + **scrobble_dict_to_db(s,update_album=update_album,dbconn=dbconn) ) for s in scrobbleslist ] @@ -285,21 +368,68 @@ def delete_scrobble(scrobble_id,dbconn=None): return True +@connection_provider +def add_track_to_album(track_id,album_id,replace=False,dbconn=None): + + conditions = [ + DB['tracks'].c.id == track_id + ] + if not replace: + # if we dont want replacement, just update if there is no album yet + conditions.append( + DB['tracks'].c.album_id == None + ) + + op = DB['tracks'].update().where( + *conditions + ).values( + album_id=album_id + ) + + result = dbconn.execute(op) + + invalidate_entity_cache() # because album info has changed + #invalidate_caches() # changing album info of tracks will change album charts + # ARE YOU FOR REAL + # it just took me like 3 hours to figure out that this one line makes the artist page load slow because + # we call this func with every new scrobble that contains album info, even if we end up not changing the album + # of course i was always debugging with the manual scrobble button which just doesnt send any album info + # and because we expel all caches every single time, the artist page then needs to recalculate the weekly charts of + # ALL OF RECORDED HISTORY in order to display top weeks + # lmao + # TODO: figure out something better + + + return True + +@connection_provider +def add_tracks_to_albums(track_to_album_id_dict,replace=False,dbconn=None): + + for track_id in track_to_album_id_dict: + add_track_to_album(track_id,track_to_album_id_dict[track_id],dbconn=dbconn) + +@connection_provider +def remove_album(*track_ids,dbconn=None): + + DB['tracks'].update().where( + DB['tracks'].c.track_id.in_(track_ids) + ).values( + album_id=None + ) + ### these will 'get' the ID of an entity, creating it if necessary @cached_wrapper @connection_provider -def get_track_id(trackdict,create_new=True,dbconn=None): +def get_track_id(trackdict,create_new=True,update_album=False,dbconn=None): ntitle = normalize_name(trackdict['title']) - artist_ids = [get_artist_id(a,dbconn=dbconn) for a in trackdict['artists']] + artist_ids = [get_artist_id(a,create_new=create_new,dbconn=dbconn) for a in trackdict['artists']] artist_ids = list(set(artist_ids)) - op = DB['tracks'].select( -# DB['tracks'].c.id - ).where( + op = DB['tracks'].select().where( DB['tracks'].c.title_normalized==ntitle ) result = dbconn.execute(op).all() @@ -310,23 +440,34 @@ def get_track_id(trackdict,create_new=True,dbconn=None): op = DB['trackartists'].select( # DB['trackartists'].c.artist_id ).where( - DB['trackartists'].c.track_id==row[0] + DB['trackartists'].c.track_id==row.id ) result = dbconn.execute(op).all() match_artist_ids = [r.artist_id for r in result] #print("required artists",artist_ids,"this match",match_artist_ids) if set(artist_ids) == set(match_artist_ids): #print("ID for",trackdict['title'],"was",row[0]) + if trackdict.get('album') and create_new: + # if we don't supply create_new, it means we just want to get info about a track + # which means no need to write album info, even if it was new + + # if we havent set update_album, we only want to assign the album in case the track + # has no album yet. this means we also only want to create a potentially new album in that case + album_id = get_album_id(trackdict['album'],create_new=(update_album or not row.album_id),dbconn=dbconn) + add_track_to_album(row.id,album_id,replace=update_album,dbconn=dbconn) + + return row.id if not create_new: return None - + #print("Creating new track") op = DB['tracks'].insert().values( **track_dict_to_db(trackdict,dbconn=dbconn) ) result = dbconn.execute(op) track_id = result.inserted_primary_key[0] + #print(track_id) for artist_id in artist_ids: op = DB['trackartists'].insert().values( @@ -335,6 +476,9 @@ def get_track_id(trackdict,create_new=True,dbconn=None): ) result = dbconn.execute(op) #print("Created",trackdict['title'],track_id) + + if trackdict.get('album'): + add_track_to_album(track_id,get_album_id(trackdict['album'],dbconn=dbconn),dbconn=dbconn) return track_id @cached_wrapper @@ -343,9 +487,7 @@ def get_artist_id(artistname,create_new=True,dbconn=None): nname = normalize_name(artistname) #print("looking for",nname) - op = DB['artists'].select( -# DB['artists'].c.id - ).where( + op = DB['artists'].select().where( DB['artists'].c.name_normalized==nname ) result = dbconn.execute(op).all() @@ -364,6 +506,59 @@ def get_artist_id(artistname,create_new=True,dbconn=None): return result.inserted_primary_key[0] +@cached_wrapper +@connection_provider +def get_album_id(albumdict,create_new=True,ignore_albumartists=False,dbconn=None): + ntitle = normalize_name(albumdict['albumtitle']) + artist_ids = [get_artist_id(a,dbconn=dbconn) for a in (albumdict.get('artists') or [])] + artist_ids = list(set(artist_ids)) + + op = DB['albums'].select( +# DB['albums'].c.id + ).where( + DB['albums'].c.albtitle_normalized==ntitle + ) + result = dbconn.execute(op).all() + for row in result: + if ignore_albumartists: + return row.id + else: + # check if the artists are the same + foundtrackartists = [] + + op = DB['albumartists'].select( + # DB['albumartists'].c.artist_id + ).where( + DB['albumartists'].c.album_id==row.id + ) + result = dbconn.execute(op).all() + match_artist_ids = [r.artist_id for r in result] + #print("required artists",artist_ids,"this match",match_artist_ids) + if set(artist_ids) == set(match_artist_ids): + #print("ID for",albumdict['title'],"was",row[0]) + return row.id + + if not create_new: return None + + + op = DB['albums'].insert().values( + **album_dict_to_db(albumdict,dbconn=dbconn) + ) + result = dbconn.execute(op) + album_id = result.inserted_primary_key[0] + + for artist_id in artist_ids: + op = DB['albumartists'].insert().values( + album_id=album_id, + artist_id=artist_id + ) + result = dbconn.execute(op) + #print("Created",trackdict['title'],track_id) + return album_id + + + + ### Edit existing @@ -385,7 +580,7 @@ def edit_scrobble(scrobble_id,scrobbleupdatedict,dbconn=None): dbconn.execute(op) - +# edit function only for primary db information (not linked fields) @connection_provider def edit_artist(id,artistupdatedict,dbconn=None): @@ -408,6 +603,7 @@ def edit_artist(id,artistupdatedict,dbconn=None): return True +# edit function only for primary db information (not linked fields) @connection_provider def edit_track(id,trackupdatedict,dbconn=None): @@ -430,6 +626,111 @@ def edit_track(id,trackupdatedict,dbconn=None): return True +# edit function only for primary db information (not linked fields) +@connection_provider +def edit_album(id,albumupdatedict,dbconn=None): + + album = get_album(id,dbconn=dbconn) + changedalbum = {**album,**albumupdatedict} + + dbentry = album_dict_to_db(albumupdatedict,dbconn=dbconn) + dbentry = {k:v for k,v in dbentry.items() if v} + + existing_album_id = get_album_id(changedalbum,create_new=False,dbconn=dbconn) + if existing_album_id not in (None,id): + raise exc.TrackExists(changedalbum) + + op = DB['albums'].update().where( + DB['albums'].c.id==id + ).values( + **dbentry + ) + result = dbconn.execute(op) + + return True + + +### Edit associations + +@connection_provider +def add_artists_to_tracks(track_ids,artist_ids,dbconn=None): + + op = DB['trackartists'].insert().values([ + {'track_id':track_id,'artist_id':artist_id} + for track_id in track_ids for artist_id in artist_ids + ]) + + result = dbconn.execute(op) + + # the resulting tracks could now be duplicates of existing ones + # this also takes care of clean_db + merge_duplicate_tracks(dbconn=dbconn) + + return True + +@connection_provider +def remove_artists_from_tracks(track_ids,artist_ids,dbconn=None): + + # only tracks that have at least one other artist + subquery = DB['trackartists'].select().where( + ~DB['trackartists'].c.artist_id.in_(artist_ids) + ).with_only_columns( + DB['trackartists'].c.track_id + ).distinct().alias('sub') + + op = DB['trackartists'].delete().where( + sql.and_( + DB['trackartists'].c.track_id.in_(track_ids), + DB['trackartists'].c.artist_id.in_(artist_ids), + DB['trackartists'].c.track_id.in_(subquery.select()) + ) + ) + + result = dbconn.execute(op) + + # the resulting tracks could now be duplicates of existing ones + # this also takes care of clean_db + merge_duplicate_tracks(dbconn=dbconn) + + return True + + +@connection_provider +def add_artists_to_albums(album_ids,artist_ids,dbconn=None): + + op = DB['albumartists'].insert().values([ + {'album_id':album_id,'artist_id':artist_id} + for album_id in album_ids for artist_id in artist_ids + ]) + + result = dbconn.execute(op) + + # the resulting albums could now be duplicates of existing ones + # this also takes care of clean_db + merge_duplicate_albums(dbconn=dbconn) + + return True + + +@connection_provider +def remove_artists_from_albums(album_ids,artist_ids,dbconn=None): + + # no check here, albums are allowed to have zero artists + + op = DB['albumartists'].delete().where( + sql.and_( + DB['albumartists'].c.album_id.in_(album_ids), + DB['albumartists'].c.artist_id.in_(artist_ids) + ) + ) + + result = dbconn.execute(op) + + # the resulting albums could now be duplicates of existing ones + # this also takes care of clean_db + merge_duplicate_albums(dbconn=dbconn) + + return True ### Merge @@ -475,6 +776,28 @@ def merge_artists(target_id,source_ids,dbconn=None): result = dbconn.execute(op) + + # same for albums + op = DB['albumartists'].select().where( + DB['albumartists'].c.artist_id.in_(source_ids + [target_id]) + ) + result = dbconn.execute(op) + + album_ids = set(row.album_id for row in result) + + op = DB['albumartists'].delete().where( + DB['albumartists'].c.artist_id.in_(source_ids + [target_id]), + ) + result = dbconn.execute(op) + + op = DB['albumartists'].insert().values([ + {'album_id':album_id,'artist_id':target_id} + for album_id in album_ids + ]) + + result = dbconn.execute(op) + + # tracks_artists = {} # for row in result: # tracks_artists.setdefault(row.track_id,[]).append(row.artist_id) @@ -490,34 +813,68 @@ def merge_artists(target_id,source_ids,dbconn=None): # ) # result = dbconn.execute(op) - # this could have created duplicate tracks + # this could have created duplicate tracks and albums merge_duplicate_tracks(artist_id=target_id,dbconn=dbconn) + merge_duplicate_albums(artist_id=target_id,dbconn=dbconn) clean_db(dbconn=dbconn) return True +@connection_provider +def merge_albums(target_id,source_ids,dbconn=None): + + op = DB['tracks'].update().where( + DB['tracks'].c.album_id.in_(source_ids) + ).values( + album_id=target_id + ) + result = dbconn.execute(op) + clean_db(dbconn=dbconn) + + return True + ### Functions that get rows according to parameters @cached_wrapper @connection_provider -def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,dbconn=None): +def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,limit=None,reverse=False,associated=False,dbconn=None): if since is None: since=0 if to is None: to=now() - artist_id = get_artist_id(artist,dbconn=dbconn) + if associated: + artist_ids = get_associated_artists(artist,resolve_ids=False,dbconn=dbconn) + [get_artist_id(artist,create_new=False,dbconn=dbconn)] + else: + artist_ids = [get_artist_id(artist,create_new=False,dbconn=dbconn)] + jointable = sql.join(DB['scrobbles'],DB['trackartists'],DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id) op = jointable.select().where( - DB['scrobbles'].c.timestamp<=to, - DB['scrobbles'].c.timestamp>=since, - DB['trackartists'].c.artist_id==artist_id - ).order_by(sql.asc('timestamp')) + DB['scrobbles'].c.timestamp.between(since,to), + DB['trackartists'].c.artist_id.in_(artist_ids) + ) + if reverse: + op = op.order_by(sql.desc('timestamp')) + else: + op = op.order_by(sql.asc('timestamp')) + if limit: + op = op.limit(limit) result = dbconn.execute(op).all() + # remove duplicates (multiple associated artists in the song, e.g. Irene & Seulgi being both counted as Red Velvet) + # distinct on doesn't seem to exist in sqlite + seen = set() + filtered_result = [] + for row in result: + if row.timestamp not in seen: + filtered_result.append(row) + seen.add(row.timestamp) + result = filtered_result + + if resolve_references: result = scrobbles_db_to_dict(result,dbconn=dbconn) #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for row in result] @@ -525,18 +882,25 @@ def get_scrobbles_of_artist(artist,since=None,to=None,resolve_references=True,db @cached_wrapper @connection_provider -def get_scrobbles_of_track(track,since=None,to=None,resolve_references=True,dbconn=None): +def get_scrobbles_of_track(track,since=None,to=None,resolve_references=True,limit=None,reverse=False,dbconn=None): if since is None: since=0 if to is None: to=now() - track_id = get_track_id(track,dbconn=dbconn) + track_id = get_track_id(track,create_new=False,dbconn=dbconn) + op = DB['scrobbles'].select().where( DB['scrobbles'].c.timestamp<=to, DB['scrobbles'].c.timestamp>=since, DB['scrobbles'].c.track_id==track_id - ).order_by(sql.asc('timestamp')) + ) + if reverse: + op = op.order_by(sql.desc('timestamp')) + else: + op = op.order_by(sql.asc('timestamp')) + if limit: + op = op.limit(limit) result = dbconn.execute(op).all() if resolve_references: @@ -546,20 +910,59 @@ def get_scrobbles_of_track(track,since=None,to=None,resolve_references=True,dbco @cached_wrapper @connection_provider -def get_scrobbles(since=None,to=None,resolve_references=True,dbconn=None): +def get_scrobbles_of_album(album,since=None,to=None,resolve_references=True,limit=None,reverse=False,dbconn=None): + + if since is None: since=0 + if to is None: to=now() + + album_id = get_album_id(album,create_new=False,dbconn=dbconn) + + jointable = sql.join(DB['scrobbles'],DB['tracks'],DB['scrobbles'].c.track_id == DB['tracks'].c.id) + + op = jointable.select().where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['tracks'].c.album_id==album_id + ) + if reverse: + op = op.order_by(sql.desc('timestamp')) + else: + op = op.order_by(sql.asc('timestamp')) + if limit: + op = op.limit(limit) + result = dbconn.execute(op).all() + + if resolve_references: + result = scrobbles_db_to_dict(result) + #result = [scrobble_db_to_dict(row) for row in result] + return result + +@cached_wrapper +@connection_provider +def get_scrobbles(since=None,to=None,resolve_references=True,limit=None,reverse=False,dbconn=None): + if since is None: since=0 if to is None: to=now() op = DB['scrobbles'].select().where( - DB['scrobbles'].c.timestamp<=to, - DB['scrobbles'].c.timestamp>=since, - ).order_by(sql.asc('timestamp')) + DB['scrobbles'].c.timestamp.between(since,to) + ) + if reverse: + op = op.order_by(sql.desc('timestamp')) + else: + op = op.order_by(sql.asc('timestamp')) + if limit: + op = op.limit(limit) + + result = dbconn.execute(op).all() if resolve_references: result = scrobbles_db_to_dict(result,dbconn=dbconn) #result = [scrobble_db_to_dict(row,resolve_references=resolve_references) for i,row in enumerate(result) if i=since, + DB['scrobbles'].c.timestamp.between(since,to) ) result = dbconn.execute(op).all() @@ -623,11 +1025,20 @@ def get_tracks(dbconn=None): return tracks_db_to_dict(result,dbconn=dbconn) +@cached_wrapper +@connection_provider +def get_albums(dbconn=None): + + op = DB['albums'].select() + result = dbconn.execute(op).all() + + return albums_db_to_dict(result,dbconn=dbconn) + ### functions that count rows for parameters @cached_wrapper @connection_provider -def count_scrobbles_by_artist(since,to,resolve_ids=True,dbconn=None): +def count_scrobbles_by_artist(since,to,associated=True,resolve_ids=True,dbconn=None): jointable = sql.join( DB['scrobbles'], DB['trackartists'], @@ -640,27 +1051,35 @@ def count_scrobbles_by_artist(since,to,resolve_ids=True,dbconn=None): DB['trackartists'].c.artist_id == DB['associated_artists'].c.source_artist, isouter=True ) + + if associated: + artistselect = sql.func.coalesce(DB['associated_artists'].c.target_artist,DB['trackartists'].c.artist_id) + else: + artistselect = DB['trackartists'].c.artist_id + op = sql.select( sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), # only count distinct scrobbles - because of artist replacement, we could end up # with two artists of the same scrobble counting it twice for the same artist # e.g. Irene and Seulgi adding two scrobbles to Red Velvet for one real scrobble - sql.func.coalesce(DB['associated_artists'].c.target_artist,DB['trackartists'].c.artist_id).label('artist_id') + artistselect.label('artist_id'), # use the replaced artist as artist to count if it exists, otherwise original one + sql.func.sum( + sql.case((DB['trackartists'].c.artist_id == artistselect, 1), else_=0) + ).label('really_by_this_artist') + # also select the original artist in any case as a separate column ).select_from(jointable2).where( - DB['scrobbles'].c.timestamp<=to, - DB['scrobbles'].c.timestamp>=since + DB['scrobbles'].c.timestamp.between(since,to) ).group_by( - sql.func.coalesce(DB['associated_artists'].c.target_artist,DB['trackartists'].c.artist_id) + artistselect ).order_by(sql.desc('count')) result = dbconn.execute(op).all() if resolve_ids: - counts = [row.count for row in result] artists = get_artists_map([row.artist_id for row in result],dbconn=dbconn) - result = [{'scrobbles':row.count,'artist':artists[row.artist_id]} for row in result] + result = [{'scrobbles':row.count,'real_scrobbles':row.really_by_this_artist,'artist':artists[row.artist_id],'artist_id':row.artist_id} for row in result] else: - result = [{'scrobbles':row.count,'artist_id':row.artist_id} for row in result] + result = [{'scrobbles':row.count,'real_scrobbles':row.really_by_this_artist,'artist_id':row.artist_id} for row in result] result = rank(result,key='scrobbles') return result @@ -679,9 +1098,8 @@ def count_scrobbles_by_track(since,to,resolve_ids=True,dbconn=None): result = dbconn.execute(op).all() if resolve_ids: - counts = [row.count for row in result] tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) - result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + result = [{'scrobbles':row.count,'track':tracks[row.track_id],'track_id':row.track_id} for row in result] else: result = [{'scrobbles':row.count,'track_id':row.track_id} for row in result] result = rank(result,key='scrobbles') @@ -689,9 +1107,184 @@ def count_scrobbles_by_track(since,to,resolve_ids=True,dbconn=None): @cached_wrapper @connection_provider -def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None): +def count_scrobbles_by_album(since,to,resolve_ids=True,dbconn=None): - artist_id = get_artist_id(artist,dbconn=dbconn) + jointable = sql.join( + DB['scrobbles'], + DB['tracks'], + DB['scrobbles'].c.track_id == DB['tracks'].c.id + ) + + op = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + DB['tracks'].c.album_id + ).select_from(jointable).where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['tracks'].c.album_id != None + ).group_by(DB['tracks'].c.album_id).order_by(sql.desc('count')) + result = dbconn.execute(op).all() + + if resolve_ids: + albums = get_albums_map([row.album_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'album':albums[row.album_id],'album_id':row.album_id} for row in result] + else: + result = [{'scrobbles':row.count,'album_id':row.album_id} for row in result] + result = rank(result,key='scrobbles') + return result + + +# get ALL albums the artist is in any way related to and rank them by TBD +@cached_wrapper +@connection_provider +def count_scrobbles_by_album_combined(since,to,artist,associated=False,resolve_ids=True,dbconn=None): + + if associated: + artist_ids = get_associated_artists(artist,resolve_ids=False,dbconn=dbconn) + [get_artist_id(artist,dbconn=dbconn)] + else: + artist_ids = [get_artist_id(artist,dbconn=dbconn)] + + # get all tracks that either have a relevant trackartist + # or are on an album with a relevant albumartist + op1 = sql.select(DB['tracks'].c.id).select_from( + sql.join( + sql.join( + DB['tracks'], + DB['trackartists'], + DB['tracks'].c.id == DB['trackartists'].c.track_id + ), + DB['albumartists'], + DB['tracks'].c.album_id == DB['albumartists'].c.album_id, + isouter=True + ) + ).where( + DB['tracks'].c.album_id.is_not(None), # tracks without albums don't matter + sql.or_( + DB['trackartists'].c.artist_id.in_(artist_ids), + DB['albumartists'].c.artist_id.in_(artist_ids) + ) + ) + relevant_tracks = dbconn.execute(op1).all() + relevant_track_ids = set(row.id for row in relevant_tracks) + #for row in relevant_tracks: + # print(get_track(row.id)) + + op2 = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + DB['tracks'].c.album_id + ).select_from( + sql.join( + DB['scrobbles'], + DB['tracks'], + DB['scrobbles'].c.track_id == DB['tracks'].c.id + ) + ).where( + DB['scrobbles'].c.timestamp.between(since,to), + DB['scrobbles'].c.track_id.in_(relevant_track_ids) + ).group_by(DB['tracks'].c.album_id).order_by(sql.desc('count')) + result = dbconn.execute(op2).all() + + if resolve_ids: + albums = get_albums_map([row.album_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'album':albums[row.album_id],'album_id':row.album_id} for row in result] + else: + result = [{'scrobbles':row.count,'album_id':row.album_id} for row in result] + result = rank(result,key='scrobbles') + + #from pprint import pprint + #pprint(result) + return result + + +@cached_wrapper +@connection_provider +# this ranks the albums of that artist, not albums the artist appears on - even scrobbles +# of tracks the artist is not part of! +def count_scrobbles_by_album_of_artist(since,to,artist,associated=False,resolve_ids=True,dbconn=None): + + if associated: + artist_ids = get_associated_artists(artist,resolve_ids=False,dbconn=dbconn) + [get_artist_id(artist,dbconn=dbconn)] + else: + artist_ids = [get_artist_id(artist,dbconn=dbconn)] + + jointable = sql.join( + DB['scrobbles'], + DB['tracks'], + DB['scrobbles'].c.track_id == DB['tracks'].c.id + ) + jointable2 = sql.join( + jointable, + DB['albumartists'], + DB['tracks'].c.album_id == DB['albumartists'].c.album_id + ) + + op = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + DB['tracks'].c.album_id + ).select_from(jointable2).where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['albumartists'].c.artist_id.in_(artist_ids) + ).group_by(DB['tracks'].c.album_id).order_by(sql.desc('count')) + result = dbconn.execute(op).all() + + if resolve_ids: + albums = get_albums_map([row.album_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'album':albums[row.album_id],'album_id':row.album_id} for row in result] + else: + result = [{'scrobbles':row.count,'album_id':row.album_id} for row in result] + result = rank(result,key='scrobbles') + return result + +@cached_wrapper +@connection_provider +# this ranks the tracks of that artist by the album they appear on - even when the album +# is not the artist's +def count_scrobbles_of_artist_by_album(since,to,artist,associated=False,resolve_ids=True,dbconn=None): + + if associated: + artist_ids = get_associated_artists(artist,resolve_ids=False,dbconn=dbconn) + [get_artist_id(artist,dbconn=dbconn)] + else: + artist_ids = [get_artist_id(artist,dbconn=dbconn)] + + jointable = sql.join( + DB['scrobbles'], + DB['trackartists'], + DB['scrobbles'].c.track_id == DB['trackartists'].c.track_id + ) + jointable2 = sql.join( + jointable, + DB['tracks'], + DB['scrobbles'].c.track_id == DB['tracks'].c.id + ) + + op = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + DB['tracks'].c.album_id + ).select_from(jointable2).where( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['trackartists'].c.artist_id.in_(artist_ids) + ).group_by(DB['tracks'].c.album_id).order_by(sql.desc('count')) + result = dbconn.execute(op).all() + + if resolve_ids: + albums = get_albums_map([row.album_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'album':albums[row.album_id],'album_id':row.album_id} for row in result if row.album_id] + else: + result = [{'scrobbles':row.count,'album_id':row.album_id} for row in result] + result = rank(result,key='scrobbles') + return result + + +@cached_wrapper +@connection_provider +def count_scrobbles_by_track_of_artist(since,to,artist,associated=False,resolve_ids=True,dbconn=None): + + if associated: + artist_ids = get_associated_artists(artist,resolve_ids=False,dbconn=dbconn) + [get_artist_id(artist,dbconn=dbconn)] + else: + artist_ids = [get_artist_id(artist,dbconn=dbconn)] jointable = sql.join( DB['scrobbles'], @@ -705,18 +1298,51 @@ def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None): ).select_from(jointable).filter( DB['scrobbles'].c.timestamp<=to, DB['scrobbles'].c.timestamp>=since, - DB['trackartists'].c.artist_id==artist_id + DB['trackartists'].c.artist_id.in_(artist_ids) ).group_by(DB['scrobbles'].c.track_id).order_by(sql.desc('count')) result = dbconn.execute(op).all() - counts = [row.count for row in result] - tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) - result = [{'scrobbles':row.count,'track':tracks[row.track_id]} for row in result] + if resolve_ids: + tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'track':tracks[row.track_id],'track_id':row.track_id} for row in result] + else: + result = [{'scrobbles':row.count,'track_id':row.track_id} for row in result] result = rank(result,key='scrobbles') return result +@cached_wrapper +@connection_provider +def count_scrobbles_by_track_of_album(since,to,album,resolve_ids=True,dbconn=None): + + album_id = get_album_id(album,dbconn=dbconn) if album else None + + jointable = sql.join( + DB['scrobbles'], + DB['tracks'], + DB['scrobbles'].c.track_id == DB['tracks'].c.id + ) + + op = sql.select( + sql.func.count(sql.func.distinct(DB['scrobbles'].c.timestamp)).label('count'), + DB['scrobbles'].c.track_id + ).select_from(jointable).filter( + DB['scrobbles'].c.timestamp<=to, + DB['scrobbles'].c.timestamp>=since, + DB['tracks'].c.album_id==album_id + ).group_by(DB['scrobbles'].c.track_id).order_by(sql.desc('count')) + result = dbconn.execute(op).all() + + + if resolve_ids: + tracks = get_tracks_map([row.track_id for row in result],dbconn=dbconn) + result = [{'scrobbles':row.count,'track':tracks[row.track_id],'track_id':row.track_id} for row in result] + else: + result = [{'scrobbles':row.count,'track_id':row.track_id} for row in result] + result = rank(result,key='scrobbles') + return result + ### functions that get mappings for several entities -> rows @@ -724,7 +1350,18 @@ def count_scrobbles_by_track_of_artist(since,to,artist,dbconn=None): @cached_wrapper_individual @connection_provider def get_artists_of_tracks(track_ids,dbconn=None): - op = sql.join(DB['trackartists'],DB['artists']).select().where( + + jointable = sql.join( + DB['trackartists'], + DB['artists'] + ) + + # we need to select to avoid multiple 'id' columns that will then + # be misinterpreted by the row-dict converter + op = sql.select( + DB['artists'], + DB['trackartists'].c.track_id + ).select_from(jointable).where( DB['trackartists'].c.track_id.in_(track_ids) ) result = dbconn.execute(op).all() @@ -734,6 +1371,89 @@ def get_artists_of_tracks(track_ids,dbconn=None): artists.setdefault(row.track_id,[]).append(artist_db_to_dict(row,dbconn=dbconn)) return artists +@cached_wrapper_individual +@connection_provider +def get_artists_of_albums(album_ids,dbconn=None): + + jointable = sql.join( + DB['albumartists'], + DB['artists'] + ) + + # we need to select to avoid multiple 'id' columns that will then + # be misinterpreted by the row-dict converter + op = sql.select( + DB['artists'], + DB['albumartists'].c.album_id + ).select_from(jointable).where( + DB['albumartists'].c.album_id.in_(album_ids) + ) + result = dbconn.execute(op).all() + + artists = {} + for row in result: + artists.setdefault(row.album_id,[]).append(artist_db_to_dict(row,dbconn=dbconn)) + return artists + +@cached_wrapper_individual +@connection_provider +def get_albums_of_artists(artist_ids,dbconn=None): + + jointable = sql.join( + DB['albumartists'], + DB['albums'] + ) + + # we need to select to avoid multiple 'id' columns that will then + # be misinterpreted by the row-dict converter + op = sql.select( + DB["albums"], + DB['albumartists'].c.artist_id + ).select_from(jointable).where( + DB['albumartists'].c.artist_id.in_(artist_ids) + ) + result = dbconn.execute(op).all() + + albums = {} + for row in result: + albums.setdefault(row.artist_id,[]).append(album_db_to_dict(row,dbconn=dbconn)) + return albums + +@cached_wrapper_individual +@connection_provider +# this includes the artists' own albums! +def get_albums_artists_appear_on(artist_ids,dbconn=None): + + jointable1 = sql.join( + DB["trackartists"], + DB["tracks"] + ) + jointable2 = sql.join( + jointable1, + DB["albums"] + ) + + # we need to select to avoid multiple 'id' columns that will then + # be misinterpreted by the row-dict converter + op = sql.select( + DB["albums"], + DB["trackartists"].c.artist_id + ).select_from(jointable2).where( + DB['trackartists'].c.artist_id.in_(artist_ids) + ) + result = dbconn.execute(op).all() + + albums = {} + # avoid duplicates from multiple tracks in album by same artist + already_done = {} + for row in result: + if row.id in already_done.setdefault(row.artist_id,[]): + pass + else: + albums.setdefault(row.artist_id,[]).append(album_db_to_dict(row,dbconn=dbconn)) + already_done[row.artist_id].append(row.id) + return albums + @cached_wrapper_individual @connection_provider @@ -769,11 +1489,27 @@ def get_artists_map(artist_ids,dbconn=None): return artists +@cached_wrapper_individual +@connection_provider +def get_albums_map(album_ids,dbconn=None): + op = DB['albums'].select().where( + DB['albums'].c.id.in_(album_ids) + ) + result = dbconn.execute(op).all() + + albums = {} + result = list(result) + # this will get a list of albumdicts in the correct order of our rows + albumdicts = albums_db_to_dict(result,dbconn=dbconn) + for row,albumdict in zip(result,albumdicts): + albums[row.id] = albumdict + return albums + ### associations @cached_wrapper @connection_provider -def get_associated_artists(*artists,dbconn=None): +def get_associated_artists(*artists,resolve_ids=True,dbconn=None): artist_ids = [get_artist_id(a,dbconn=dbconn) for a in artists] jointable = sql.join( @@ -782,13 +1518,60 @@ def get_associated_artists(*artists,dbconn=None): DB['associated_artists'].c.source_artist == DB['artists'].c.id ) - op = jointable.select().where( + # we need to select to avoid multiple 'id' columns that will then + # be misinterpreted by the row-dict converter + op = sql.select( + DB['artists'] + ).select_from(jointable).where( DB['associated_artists'].c.target_artist.in_(artist_ids) ) result = dbconn.execute(op).all() - artists = artists_db_to_dict(result,dbconn=dbconn) - return artists + if resolve_ids: + artists = artists_db_to_dict(result,dbconn=dbconn) + return artists + else: + return [a.id for a in result] + +@cached_wrapper +@connection_provider +def get_associated_artist_map(artists=[],artist_ids=None,resolve_ids=True,dbconn=None): + + ids_supplied = (artist_ids is not None) + + if not ids_supplied: + artist_ids = [get_artist_id(a,dbconn=dbconn) for a in artists] + + + jointable = sql.join( + DB['associated_artists'], + DB['artists'], + DB['associated_artists'].c.source_artist == DB['artists'].c.id + ) + + # we need to select to avoid multiple 'id' columns that will then + # be misinterpreted by the row-dict converter + op = sql.select( + DB['artists'], + DB['associated_artists'].c.target_artist + ).select_from(jointable).where( + DB['associated_artists'].c.target_artist.in_(artist_ids) + ) + result = dbconn.execute(op).all() + + artists_to_associated = {a_id:[] for a_id in artist_ids} + for row in result: + if resolve_ids: + artists_to_associated[row.target_artist].append(artists_db_to_dict([row],dbconn=dbconn)[0]) + else: + artists_to_associated[row.target_artist].append(row.id) + + if not ids_supplied: + # if we supplied the artists, we want to convert back for the result + artists_to_associated = {artists[artist_ids.index(k)]:v for k,v in artists_to_associated.items()} + + return artists_to_associated + @cached_wrapper @connection_provider @@ -801,8 +1584,11 @@ def get_credited_artists(*artists,dbconn=None): DB['associated_artists'].c.target_artist == DB['artists'].c.id ) - - op = jointable.select().where( + # we need to select to avoid multiple 'id' columns that will then + # be misinterpreted by the row-dict converter + op = sql.select( + DB['artists'] + ).select_from(jointable).where( DB['associated_artists'].c.source_artist.in_(artist_ids) ) result = dbconn.execute(op).all() @@ -835,6 +1621,16 @@ def get_artist(id,dbconn=None): artistinfo = result[0] return artist_db_to_dict(artistinfo,dbconn=dbconn) +@cached_wrapper +@connection_provider +def get_album(id,dbconn=None): + op = DB['albums'].select().where( + DB['albums'].c.id==id + ) + result = dbconn.execute(op).all() + + albuminfo = result[0] + return album_db_to_dict(albuminfo,dbconn=dbconn) @cached_wrapper @connection_provider @@ -867,33 +1663,55 @@ def search_track(searchterm,dbconn=None): return [get_track(row.id,dbconn=dbconn) for row in result] +@cached_wrapper +@connection_provider +def search_album(searchterm,dbconn=None): + op = DB['albums'].select().where( + DB['albums'].c.albtitle_normalized.ilike(normalize_name(f"%{searchterm}%")) + ) + result = dbconn.execute(op).all() + + return [get_album(row.id,dbconn=dbconn) for row in result] ##### MAINTENANCE @runhourly @connection_provider +@no_aux_mode def clean_db(dbconn=None): - log(f"Database Cleanup...") + with SCROBBLE_LOCK: + log(f"Database Cleanup...") - to_delete = [ - # tracks with no scrobbles (trackartist entries first) - "from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))", - "from tracks where id not in (select track_id from scrobbles)", - # artists with no tracks - "from artists where id not in (select artist_id from trackartists) and id not in (select target_artist from associated_artists)", - # tracks with no artists (scrobbles first) - "from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))", - "from tracks where id not in (select track_id from trackartists)" - ] + to_delete = [ + # tracks with no scrobbles (trackartist entries first) + "from trackartists where track_id in (select id from tracks where id not in (select track_id from scrobbles))", + "from tracks where id not in (select track_id from scrobbles)", + # artists with no tracks AND no albums + "from artists where id not in (select artist_id from trackartists) \ + and id not in (select target_artist from associated_artists) \ + and id not in (select artist_id from albumartists)", + # tracks with no artists (scrobbles first) + "from scrobbles where track_id in (select id from tracks where id not in (select track_id from trackartists))", + "from tracks where id not in (select track_id from trackartists)", + # albums with no tracks (albumartist entries first) + "from albumartists where album_id in (select id from albums where id not in (select album_id from tracks where album_id is not null))", + "from albums where id not in (select album_id from tracks where album_id is not null)", + # albumartist entries that are missing a reference + "from albumartists where album_id not in (select album_id from tracks where album_id is not null)", + "from albumartists where artist_id not in (select id from artists)", + # trackartist entries that mare missing a reference + "from trackartists where track_id not in (select id from tracks)", + "from trackartists where artist_id not in (select id from artists)" + ] - for d in to_delete: - selection = dbconn.execute(sql.text(f"select * {d}")) - for row in selection.all(): - log(f"Deleting {row}") - deletion = dbconn.execute(sql.text(f"delete {d}")) + for d in to_delete: + selection = dbconn.execute(sql.text(f"select * {d}")) + for row in selection.all(): + log(f"Deleting {row}") + deletion = dbconn.execute(sql.text(f"delete {d}")) - log("Database Cleanup complete!") + log("Database Cleanup complete!") @@ -906,7 +1724,9 @@ def clean_db(dbconn=None): @runmonthly +@no_aux_mode def renormalize_names(): + with SCROBBLE_LOCK: with engine.begin() as conn: rows = conn.execute(DB['artists'].select()).all() @@ -923,10 +1743,15 @@ def renormalize_names(): @connection_provider -def merge_duplicate_tracks(artist_id,dbconn=None): +def merge_duplicate_tracks(artist_id=None,dbconn=None): + + affected_track_conditions = [] + if artist_id: + affected_track_conditions = [DB['trackartists'].c.artist_id == artist_id] + rows = dbconn.execute( DB['trackartists'].select().where( - DB['trackartists'].c.artist_id == artist_id + *affected_track_conditions ) ) affected_tracks = [r.track_id for r in rows] @@ -958,12 +1783,150 @@ def merge_duplicate_tracks(artist_id,dbconn=None): +@connection_provider +def merge_duplicate_albums(artist_id=None,dbconn=None): + + affected_album_conditions = [] + if artist_id: + affected_album_conditions = [DB['albumartists'].c.artist_id == artist_id] + + rows = dbconn.execute( + DB['albumartists'].select().where( + *affected_album_conditions + ) + ) + affected_albums = [r.album_id for r in rows] + + album_artists = {} + rows = dbconn.execute( + DB['albumartists'].select().where( + DB['albumartists'].c.album_id.in_(affected_albums) + ) + ) + + + for row in rows: + album_artists.setdefault(row.album_id,[]).append(row.artist_id) + + artist_combos = {} + for album_id in album_artists: + artist_combos.setdefault(tuple(sorted(album_artists[album_id])),[]).append(album_id) + + for c in artist_combos: + if len(artist_combos[c]) > 1: + album_identifiers = {} + for album_id in artist_combos[c]: + album_identifiers.setdefault(normalize_name(get_album(album_id)['albumtitle']),[]).append(album_id) + for album in album_identifiers: + if len(album_identifiers[album]) > 1: + target,*src = album_identifiers[album] + merge_albums(target,src,dbconn=dbconn) +@connection_provider +def guess_albums(track_ids=None,replace=False,dbconn=None): + + MIN_NUM_TO_ASSIGN = 1 + + jointable = sql.join( + DB['scrobbles'], + DB['tracks'] + ) + + # get all scrobbles of the respective tracks that have some info + conditions = [ + DB['scrobbles'].c.extra.isnot(None) | DB['scrobbles'].c.rawscrobble.isnot(None) + ] + if track_ids is not None: + # only do these tracks + conditions.append( + DB['scrobbles'].c.track_id.in_(track_ids) + ) + if not replace: + # only tracks that have no album yet + conditions.append( + DB['tracks'].c.album_id.is_(None) + ) + + op = sql.select( + DB['scrobbles'] + ).select_from(jointable).where( + *conditions + ) + + result = dbconn.execute(op).all() + + # for each track, count what album info appears how often + possible_albums = {} + for row in result: + albumtitle, albumartists = None, None + if row.extra: + extrainfo = json.loads(row.extra) + albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title") + albumartists = extrainfo.get("album_artists",[]) + if not albumtitle: + # either we didn't have info in the exta col, or there was no albumtitle + # try the raw scrobble + extrainfo = json.loads(row.rawscrobble) + albumtitle = extrainfo.get("album_name") or extrainfo.get("album_title") + albumartists = albumartists or extrainfo.get("album_artists",[]) + if albumtitle: + hashable_albuminfo = tuple([*albumartists,albumtitle]) + possible_albums.setdefault(row.track_id,{}).setdefault(hashable_albuminfo,0) + possible_albums[row.track_id][hashable_albuminfo] += 1 + + res = {} + for track_id in possible_albums: + options = possible_albums[track_id] + if len(options)>0: + # pick the one with most occurences + mostnum = max(options[albuminfo] for albuminfo in options) + if mostnum >= MIN_NUM_TO_ASSIGN: + bestpick = [albuminfo for albuminfo in options if options[albuminfo] == mostnum][0] + #print("best pick",track_id,bestpick) + *artists,title = bestpick + res[track_id] = {"assigned":{ + "artists":artists, + "albumtitle": title + }} + if len(artists) == 0: + # for albums without artist, assume track artist + res[track_id]["guess_artists"] = [] + else: + res[track_id] = {"assigned":False,"reason":"Not enough data"} + + else: + res[track_id] = {"assigned":False,"reason":"No scrobbles with album information found"} + + + + missing_artists = [track_id for track_id in res if "guess_artists" in res[track_id]] + + #we're pointlessly getting the albumartist names here even though the IDs would be enough + #but it's better for function separation I guess + jointable = sql.join( + DB['trackartists'], + DB['artists'] + ) + op = sql.select( + DB['trackartists'].c.track_id, + DB['artists'] + ).select_from(jointable).where( + DB['trackartists'].c.track_id.in_(missing_artists) + ) + result = dbconn.execute(op).all() + + for row in result: + res[row.track_id]["guess_artists"].append(row.name) + + return res + + + ##### AUX FUNCS diff --git a/maloja/dev/profiler.py b/maloja/dev/profiler.py index 8d41455..c20db90 100644 --- a/maloja/dev/profiler.py +++ b/maloja/dev/profiler.py @@ -8,32 +8,59 @@ from doreah.timing import Clock from ..pkg_global.conf import data_dir -profiler = cProfile.Profile() + FULL_PROFILE = False +SINGLE_CALLS = False +# only save the last single call instead of adding up all calls +# of that function for more representative performance result + +if not SINGLE_CALLS: + profilers = {} + times = {} def profile(func): - def newfunc(*args,**kwargs): - if FULL_PROFILE: - benchmarkfolder = data_dir['logs']("benchmarks") - os.makedirs(benchmarkfolder,exist_ok=True) + realfunc = func + while hasattr(realfunc, '__innerfunc__'): + realfunc = realfunc.__innerfunc__ + + def newfunc(*args,**kwargs): clock = Clock() clock.start() if FULL_PROFILE: - profiler.enable() - result = func(*args,**kwargs) - if FULL_PROFILE: - profiler.disable() + benchmarkfolder = data_dir['logs']("benchmarks") + os.makedirs(benchmarkfolder,exist_ok=True) + if SINGLE_CALLS: + localprofiler = cProfile.Profile() + else: + localprofiler = profilers.setdefault(realfunc,cProfile.Profile()) + localprofiler.enable() + + result = func(*args,**kwargs) - log(f"Executed {func.__name__} ({args}, {kwargs}) in {clock.stop():.2f}s",module="debug_performance") if FULL_PROFILE: + localprofiler.disable() + + seconds = clock.stop() + + if not SINGLE_CALLS: + times.setdefault(realfunc,[]).append(seconds) + + if SINGLE_CALLS: + log(f"Executed {realfunc.__name__} ({args}, {kwargs}) in {seconds:.3f}s",module="debug_performance") + else: + log(f"Executed {realfunc.__name__} ({args}, {kwargs}) in {seconds:.3f}s (Average: { sum(times[realfunc])/len(times[realfunc]):.3f}s)",module="debug_performance") + + if FULL_PROFILE: + targetfilename = os.path.join(benchmarkfolder,f"{realfunc.__name__}.stats") try: - pstats.Stats(profiler).dump_stats(os.path.join(benchmarkfolder,f"{func.__name__}.stats")) + pstats.Stats(localprofiler).dump_stats(targetfilename) + log(f"Saved benchmark as {targetfilename}") except Exception: - pass + log(f"Failed to save benchmark as {targetfilename}") return result diff --git a/maloja/images.py b/maloja/images.py index f1c569a..2025521 100644 --- a/maloja/images.py +++ b/maloja/images.py @@ -12,195 +12,324 @@ import base64 import requests import datauri import io -from threading import Thread, Timer, BoundedSemaphore +from threading import Lock +from concurrent.futures import ThreadPoolExecutor import re import datetime +import time import sqlalchemy as sql +MAX_RESOLVE_THREADS = 5 +MAX_SECONDS_TO_RESOLVE_REQUEST = 5 + + +# remove old db file (columns missing) +try: + os.remove(data_dir['cache']('images.sqlite')) +except: + pass DB = {} -engine = sql.create_engine(f"sqlite:///{data_dir['cache']('images.sqlite')}", echo = False) +engine = sql.create_engine(f"sqlite:///{data_dir['cache']('imagecache.sqlite')}", echo = False) meta = sql.MetaData() +dblock = Lock() + DB['artists'] = sql.Table( 'artists', meta, sql.Column('id',sql.Integer,primary_key=True), sql.Column('url',sql.String), sql.Column('expire',sql.Integer), - sql.Column('raw',sql.String) +# sql.Column('raw',sql.String) + sql.Column('local',sql.Boolean), + sql.Column('localproxyurl',sql.String) ) DB['tracks'] = sql.Table( 'tracks', meta, sql.Column('id',sql.Integer,primary_key=True), sql.Column('url',sql.String), sql.Column('expire',sql.Integer), - sql.Column('raw',sql.String) +# sql.Column('raw',sql.String) + sql.Column('local',sql.Boolean), + sql.Column('localproxyurl',sql.String) +) +DB['albums'] = sql.Table( + 'albums', meta, + sql.Column('id',sql.Integer,primary_key=True), + sql.Column('url',sql.String), + sql.Column('expire',sql.Integer), +# sql.Column('raw',sql.String) + sql.Column('local',sql.Boolean), + sql.Column('localproxyurl',sql.String) ) meta.create_all(engine) -def get_image_from_cache(id,table): +def get_id_and_table(track_id=None,artist_id=None,album_id=None): + if track_id: + return track_id,'tracks' + elif album_id: + return album_id,'albums' + elif artist_id: + return artist_id,'artists' + +def get_image_from_cache(track_id=None,artist_id=None,album_id=None): now = int(datetime.datetime.now().timestamp()) + entity_id, table = get_id_and_table(track_id=track_id,artist_id=artist_id,album_id=album_id) + with engine.begin() as conn: op = DB[table].select().where( - DB[table].c.id==id, + DB[table].c.id==entity_id, DB[table].c.expire>now ) result = conn.execute(op).all() for row in result: - if row.raw is not None: - return {'type':'raw','value':row.raw} + if row.local: + return {'type':'localurl','value':row.url} + elif row.localproxyurl: + return {'type':'localurl','value':row.localproxyurl} else: - return {'type':'url','value':row.url} # returns None as value if nonexistence cached + return {'type':'url','value':row.url or None} + # value none means nonexistence is cached + # for some reason this can also be an empty string, so use or None here to unify return None # no cache entry -def set_image_in_cache(id,table,url): - remove_image_from_cache(id,table) - now = int(datetime.datetime.now().timestamp()) - if url is None: - expire = now + (malojaconfig["CACHE_EXPIRE_NEGATIVE"] * 24 * 3600) - else: - expire = now + (malojaconfig["CACHE_EXPIRE_POSITIVE"] * 24 * 3600) +def set_image_in_cache(url,track_id=None,artist_id=None,album_id=None,local=False): + remove_image_from_cache(track_id=track_id,artist_id=artist_id,album_id=album_id) + entity_id, table = get_id_and_table(track_id=track_id,artist_id=artist_id,album_id=album_id) - raw = dl_image(url) + with dblock: + now = int(datetime.datetime.now().timestamp()) + if url is None: + expire = now + (malojaconfig["CACHE_EXPIRE_NEGATIVE"] * 24 * 3600) + else: + expire = now + (malojaconfig["CACHE_EXPIRE_POSITIVE"] * 24 * 3600) - with engine.begin() as conn: - op = DB[table].insert().values( - id=id, - url=url, - expire=expire, - raw=raw - ) - result = conn.execute(op) + if not local and malojaconfig["PROXY_IMAGES"] and url is not None: + localproxyurl = dl_image(url) + else: + localproxyurl = None + + with engine.begin() as conn: + op = DB[table].insert().values( + id=entity_id, + url=url, + expire=expire, + local=local, + localproxyurl=localproxyurl + ) + result = conn.execute(op) + +def remove_image_from_cache(track_id=None,artist_id=None,album_id=None): + entity_id, table = get_id_and_table(track_id=track_id,artist_id=artist_id,album_id=album_id) + + with dblock: + with engine.begin() as conn: + op = DB[table].delete().where( + DB[table].c.id==entity_id, + ).returning( + DB[table].c.id, + DB[table].c.localproxyurl + ) + result = conn.execute(op).all() + + for row in result: + try: + targetpath = data_dir['cache']('images',row.localproxyurl.split('/')[-1]) + os.remove(targetpath) + except: + pass -def remove_image_from_cache(id,table): - with engine.begin() as conn: - op = DB[table].delete().where( - DB[table].c.id==id, - ) - result = conn.execute(op) def dl_image(url): - if not malojaconfig["PROXY_IMAGES"]: return None - if url is None: return None - if url.startswith("/"): return None #local image try: r = requests.get(url) mime = r.headers.get('content-type') or 'image/jpg' data = io.BytesIO(r.content).read() - uri = datauri.DataURI.make(mime,charset='ascii',base64=True,data=data) - log(f"Downloaded {url} for local caching") - return uri + #uri = datauri.DataURI.make(mime,charset='ascii',base64=True,data=data) + targetname = '%030x' % random.getrandbits(128) + targetpath = data_dir['cache']('images',targetname) + with open(targetpath,'wb') as fd: + fd.write(data) + return os.path.join("/cacheimages",targetname) except Exception: log(f"Image {url} could not be downloaded for local caching") return None + +resolver = ThreadPoolExecutor(max_workers=MAX_RESOLVE_THREADS,thread_name_prefix='image_resolve') + ### getting images for any website embedding now ALWAYS returns just the generic link ### even if we have already cached it, we will handle that on request def get_track_image(track=None,track_id=None): if track_id is None: - track_id = database.sqldb.get_track_id(track) + track_id = database.sqldb.get_track_id(track,create_new=False) - return f"/image?type=track&id={track_id}" + if malojaconfig["USE_ALBUM_ARTWORK_FOR_TRACKS"]: + if track is None: + track = database.sqldb.get_track(track_id) + if track.get("album"): + album_id = database.sqldb.get_album_id(track["album"]) + return get_album_image(album_id=album_id) + resolver.submit(resolve_image,track_id=track_id) + + return f"/image?track_id={track_id}" def get_artist_image(artist=None,artist_id=None): if artist_id is None: - artist_id = database.sqldb.get_artist_id(artist) + artist_id = database.sqldb.get_artist_id(artist,create_new=False) - return f"/image?type=artist&id={artist_id}" + resolver.submit(resolve_image,artist_id=artist_id) + + return f"/image?artist_id={artist_id}" + +def get_album_image(album=None,album_id=None): + if album_id is None: + album_id = database.sqldb.get_album_id(album,create_new=False) + + resolver.submit(resolve_image,album_id=album_id) + + return f"/image?album_id={album_id}" + + +# this is to keep track of what is currently being resolved +# so new requests know that they don't need to queue another resolve +image_resolve_controller_lock = Lock() +image_resolve_controller = { + 'artists':set(), + 'albums':set(), + 'tracks':set() +} + +# this function doesn't need to return any info +# it runs async to do all the work that takes time and only needs to write the result +# to the cache so the synchronous functions (http requests) can access it +def resolve_image(artist_id=None,track_id=None,album_id=None): + result = get_image_from_cache(artist_id=artist_id,track_id=track_id,album_id=album_id) + if result is not None: + # No need to do anything + return + + if artist_id: + entitytype = 'artist' + table = 'artists' + getfunc, entity_id = database.sqldb.get_artist, artist_id + elif track_id: + entitytype = 'track' + table = 'tracks' + getfunc, entity_id = database.sqldb.get_track, track_id + elif album_id: + entitytype = 'album' + table = 'albums' + getfunc, entity_id = database.sqldb.get_album, album_id -resolve_semaphore = BoundedSemaphore(8) + # is another thread already working on this? + with image_resolve_controller_lock: + if entity_id in image_resolve_controller[table]: + return + else: + image_resolve_controller[table].add(entity_id) -def resolve_track_image(track_id): - with resolve_semaphore: - # check cache - result = get_image_from_cache(track_id,'tracks') - if result is not None: - return result - track = database.sqldb.get_track(track_id) + try: + entity = getfunc(entity_id) # local image if malojaconfig["USE_LOCAL_IMAGES"]: - images = local_files(artists=track['artists'],title=track['title']) + images = local_files(**{entitytype: entity}) if len(images) != 0: result = random.choice(images) result = urllib.parse.quote(result) - result = {'type':'url','value':result} - set_image_in_cache(track_id,'tracks',result['value']) + result = {'type':'localurl','value':result} + set_image_in_cache(artist_id=artist_id,track_id=track_id,album_id=album_id,url=result['value'],local=True) return result # third party - result = thirdparty.get_image_track_all((track['artists'],track['title'])) - result = {'type':'url','value':result} - set_image_in_cache(track_id,'tracks',result['value']) + if artist_id: + result = thirdparty.get_image_artist_all(entity) + elif track_id: + result = thirdparty.get_image_track_all((entity['artists'],entity['title'])) + elif album_id: + result = thirdparty.get_image_album_all((entity['artists'],entity['albumtitle'])) - return result + result = {'type':'url','value':result or None} + set_image_in_cache(artist_id=artist_id,track_id=track_id,album_id=album_id,url=result['value']) + finally: + with image_resolve_controller_lock: + image_resolve_controller[table].remove(entity_id) -def resolve_artist_image(artist_id): - with resolve_semaphore: +# the actual http request for the full image +def image_request(artist_id=None,track_id=None,album_id=None): + + # because we use lazyload, we can allow our http requests to take a little while at least + # not the full backend request, but a few seconds to give us time to fetch some images + # because 503 retry-after doesn't seem to be honored + attempt = 0 + while attempt < MAX_SECONDS_TO_RESOLVE_REQUEST: + attempt += 1 # check cache - result = get_image_from_cache(artist_id,'artists') + result = get_image_from_cache(artist_id=artist_id,track_id=track_id,album_id=album_id) if result is not None: + # we got an entry, even if it's that there is no image (value None) + if result['value'] is None: + # use placeholder + if malojaconfig["FANCY_PLACEHOLDER_ART"]: + placeholder_url = "https://generative-placeholders.glitch.me/image?width=300&height=300&style=" + if artist_id: + result['value'] = placeholder_url + f"tiles&colors={artist_id % 100}" + if track_id: + result['value'] = placeholder_url + f"triangles&colors={track_id % 100}" + if album_id: + result['value'] = placeholder_url + f"joy-division&colors={album_id % 100}" + else: + if artist_id: + result['value'] = "/static/svg/placeholder_artist.svg" + if track_id: + result['value'] = "/static/svg/placeholder_track.svg" + if album_id: + result['value'] = "/static/svg/placeholder_album.svg" return result + time.sleep(1) - artist = database.sqldb.get_artist(artist_id) + # no entry, which means we're still working on it + return {'type':'noimage','value':'wait'} - # local image - if malojaconfig["USE_LOCAL_IMAGES"]: - images = local_files(artist=artist) - if len(images) != 0: - result = random.choice(images) - result = urllib.parse.quote(result) - result = {'type':'url','value':result} - set_image_in_cache(artist_id,'artists',result['value']) - return result - - # third party - result = thirdparty.get_image_artist_all(artist) - result = {'type':'url','value':result} - set_image_in_cache(artist_id,'artists',result['value']) - - return result # removes emojis and weird shit from names def clean(name): return "".join(c for c in name if c.isalnum() or c in []).strip() -def get_all_possible_filenames(artist=None,artists=None,title=None): - # check if we're dealing with a track or artist, then clean up names - # (only remove non-alphanumeric, allow korean and stuff) - - if title is not None and artists is not None: - track = True - title, artists = clean(title), [clean(a) for a in artists] - elif artist is not None: - track = False +# new and improved +def get_all_possible_filenames(artist=None,track=None,album=None): + if track: + title, artists = clean(track['title']), [clean(a) for a in track['artists']] + superfolder = "tracks/" + elif album: + title, artists = clean(album['albumtitle']), [clean(a) for a in album.get('artists') or []] + superfolder = "albums/" + elif artist: artist = clean(artist) - else: return [] - - - superfolder = "tracks/" if track else "artists/" + superfolder = "artists/" + else: + return [] filenames = [] - if track: - #unsafeartists = [artist.translate(None,"-_./\\") for artist in artists] + if track or album: safeartists = [re.sub("[^a-zA-Z0-9]","",artist) for artist in artists] - #unsafetitle = title.translate(None,"-_./\\") safetitle = re.sub("[^a-zA-Z0-9]","",title) if len(artists) < 4: @@ -210,7 +339,6 @@ def get_all_possible_filenames(artist=None,artists=None,title=None): unsafeperms = [sorted(artists)] safeperms = [sorted(safeartists)] - for unsafeartistlist in unsafeperms: filename = "-".join(unsafeartistlist) + "_" + title if filename != "": @@ -241,10 +369,11 @@ def get_all_possible_filenames(artist=None,artists=None,title=None): return [superfolder + name for name in filenames] -def local_files(artist=None,artists=None,title=None): + +def local_files(artist=None,album=None,track=None): - filenames = get_all_possible_filenames(artist,artists,title) + filenames = get_all_possible_filenames(artist=artist,album=album,track=track) images = [] @@ -271,13 +400,21 @@ class MalformedB64(Exception): pass def set_image(b64,**keys): - track = "title" in keys - if track: - entity = {'artists':keys['artists'],'title':keys['title']} - id = database.sqldb.get_track_id(entity) - else: - entity = keys['artist'] - id = database.sqldb.get_artist_id(entity) + if "title" in keys: + entity = {"track":keys} + id = database.sqldb.get_track_id(entity['track']) + idkeys = {'track_id':id} + dbtable = "tracks" + elif "albumtitle" in keys: + entity = {"album":keys} + id = database.sqldb.get_album_id(entity['album']) + idkeys = {'album_id':id} + dbtable = "albums" + elif "artist" in keys: + entity = keys + id = database.sqldb.get_artist_id(entity['artist']) + idkeys = {'artist_id':id} + dbtable = "artists" log("Trying to set image, b64 string: " + str(b64[:30] + "..."),module="debug") @@ -288,13 +425,13 @@ def set_image(b64,**keys): type,b64 = match.groups() b64 = base64.b64decode(b64) filename = "webupload" + str(int(datetime.datetime.now().timestamp())) + "." + type - for folder in get_all_possible_filenames(**keys): + for folder in get_all_possible_filenames(**entity): if os.path.exists(data_dir['images'](folder)): with open(data_dir['images'](folder,filename),"wb") as f: f.write(b64) break else: - folder = get_all_possible_filenames(**keys)[0] + folder = get_all_possible_filenames(**entity)[0] os.makedirs(data_dir['images'](folder)) with open(data_dir['images'](folder,filename),"wb") as f: f.write(b64) @@ -303,7 +440,6 @@ def set_image(b64,**keys): log("Saved image as " + data_dir['images'](folder,filename),module="debug") # set as current picture in rotation - if track: set_image_in_cache(id,'tracks',os.path.join("/images",folder,filename)) - else: set_image_in_cache(id,'artists',os.path.join("/images",folder,filename)) + set_image_in_cache(**idkeys,url=os.path.join("/images",folder,filename),local=True) return os.path.join("/images",folder,filename) diff --git a/maloja/jinjaenv/context.py b/maloja/jinjaenv/context.py index 908634a..0ad0710 100644 --- a/maloja/jinjaenv/context.py +++ b/maloja/jinjaenv/context.py @@ -26,8 +26,6 @@ def update_jinja_environment(): JINJA_CONTEXT = { # maloja - "db": database, #TODO: move these to connection manager as well - #"dbp":dbp, "malojatime": malojatime, "images": images, "mlj_uri": malojauri, @@ -72,6 +70,14 @@ def update_jinja_environment(): {"identifier":"longtrailing","replacekeys":{"trail":3},"localisation":"Long Trailing"}, {"identifier":"inert","replacekeys":{"trail":10},"localisation":"Inert","heavy":True}, {"identifier":"cumulative","replacekeys":{"trail":math.inf},"localisation":"Cumulative","heavy":True} + ], + "xassociated": [ + {"identifier":"include_associated","replacekeys":{"associated":True},"localisation":"Associated"}, + {"identifier":"exclude_associated","replacekeys":{"associated":False},"localisation":"Exclusive"} + ], + "xseparate": [ + {"identifier":"count_combined","replacekeys":{"separate":False},"localisation":"Combined"}, + {"identifier":"count_separate","replacekeys":{"separate":True},"localisation":"Separate"} ] } diff --git a/maloja/malojatime.py b/maloja/malojatime.py index b383a27..a808d9b 100644 --- a/maloja/malojatime.py +++ b/maloja/malojatime.py @@ -1,7 +1,7 @@ from datetime import timezone, timedelta, date, time, datetime from calendar import monthrange -from os.path import commonprefix import math +from abc import ABC, abstractmethod from .pkg_global.conf import malojaconfig @@ -28,7 +28,7 @@ def register_scrobbletime(timestamp): # Generic Time Range -class MTRangeGeneric: +class MTRangeGeneric(ABC): # despite the above, ranges that refer to the exact same real time range should evaluate as equal def __eq__(self,other): @@ -68,6 +68,15 @@ class MTRangeGeneric: def __contains__(self,timestamp): return timestamp >= self.first_stamp() and timestamp <= self.last_stamp() + @abstractmethod + def first_stamp(self): + pass + + @abstractmethod + def last_stamp(self): + pass + + # Any range that has one defining base unit, whether week, year, etc. class MTRangeSingular(MTRangeGeneric): def fromstr(self): diff --git a/maloja/malojauri.py b/maloja/malojauri.py index 6b1af6d..b2637f3 100644 --- a/maloja/malojauri.py +++ b/maloja/malojauri.py @@ -4,7 +4,7 @@ import urllib import math # this also sets defaults! -def uri_to_internal(keys,forceTrack=False,forceArtist=False,api=False): +def uri_to_internal(keys,accepted_entities=('artist','track','album'),forceTrack=False,forceArtist=False,forceAlbum=False,api=False): # output: # 1 keys that define the filtered object like artist or track @@ -12,14 +12,33 @@ def uri_to_internal(keys,forceTrack=False,forceArtist=False,api=False): # 3 keys that define interal time ranges # 4 keys that define amount limits + # if we force a type, that only means that the other types are not allowed + # it could still have no type at all (any call that isn't filtering by entity) + + if forceTrack: accepted_entities = ('track',) + if forceArtist: accepted_entities = ('artist',) + if forceAlbum: accepted_entities = ('album',) + + # API backwards compatibility + if "artist" in keys and "artist" not in accepted_entities: + if "track" in accepted_entities: + keys['trackartist'] = keys['artist'] + elif "album" in accepted_entities: + keys['albumartist'] = keys['artist'] + + # 1 - if "title" in keys and not forceArtist: - filterkeys = {"track":{"artists":keys.getall("artist"),"title":keys.get("title")}} - elif "artist" in keys and not forceTrack: - filterkeys = {"artist":keys.get("artist")} - if "associated" in keys: filterkeys["associated"] = True - else: - filterkeys = {} + filterkeys = {} + if "track" in accepted_entities and "title" in keys: + filterkeys.update({"track":{"artists":keys.getall("trackartist"),"title":keys.get("title")}}) + if "artist" in accepted_entities and "artist" in keys: + filterkeys.update({"artist": keys.get("artist"), "associated": (keys.get('associated', 'no').lower() == 'yes')}) + # associated is only used for filtering by artist, to indicate that we include associated artists + # for actual artist charts, to show that we want to count them, use 'unified' + if "album" in accepted_entities and "albumtitle" in keys: + filterkeys.update({"album":{"artists":keys.getall("albumartist"),"albumtitle":keys.get("albumtitle")}}) + + # 2 limitkeys = {} @@ -51,11 +70,20 @@ def uri_to_internal(keys,forceTrack=False,forceArtist=False,api=False): #different max than the internal one! the user doesn't get to disable pagination if "page" in keys: amountkeys["page"] = int(keys["page"]) if "perpage" in keys: amountkeys["perpage"] = int(keys["perpage"]) - + #amountkeys["reverse"] = (keys.get("reverse","no").lower() == "yes") + # we have different defaults for different things, so here we need to actually pass true false or nothing dependent + # on whether its specified + if keys.get("reverse","").lower() == 'yes': amountkeys['reverse'] = True + elif keys.get("reverse","").lower() == 'no': amountkeys['reverse'] = False #5 specialkeys = {} - if "remote" in keys: specialkeys["remote"] = keys["remote"] + #if "remote" in keys: specialkeys["remote"] = keys["remote"] + specialkeys["separate"] = (keys.get('separate','no').lower() == 'yes') + for k in keys: + if k in ['remote','b64']: + # TODO: better solution! + specialkeys[k] = keys[k] return filterkeys, limitkeys, delimitkeys, amountkeys, specialkeys @@ -80,10 +108,15 @@ def internal_to_uri(keys): if "artist" in keys: urikeys.append("artist",keys["artist"]) if keys.get("associated"): urikeys.append("associated","yes") - elif "track" in keys: + if "track" in keys: for a in keys["track"]["artists"]: - urikeys.append("artist",a) + urikeys.append("trackartist",a) urikeys.append("title",keys["track"]["title"]) + if "album" in keys: + for a in keys["album"].get("artists") or []: + urikeys.append("albumartist",a) + urikeys.append("albumtitle",keys["album"]["albumtitle"]) + #time if "timerange" in keys: @@ -116,6 +149,11 @@ def internal_to_uri(keys): urikeys.append("page",str(keys["page"])) if "perpage" in keys: urikeys.append("perpage",str(keys["perpage"])) + if "reverse" in keys: + urikeys.append("reverse","yes" if keys['reverse'] else "no") + + if keys.get("separate",False): + urikeys.append("separate","yes") return urikeys diff --git a/maloja/pkg_global/conf.py b/maloja/pkg_global/conf.py index ea6a8c0..d7f2a66 100644 --- a/maloja/pkg_global/conf.py +++ b/maloja/pkg_global/conf.py @@ -6,6 +6,8 @@ from doreah.configuration import types as tp from ..__pkginfo__ import VERSION +# this mode specifies whether we run some auxiliary task instead of the main server +AUX_MODE = True # if DATA_DIRECTORY is specified, this is the directory to use for EVERYTHING, no matter what @@ -148,14 +150,17 @@ malojaconfig = Configuration( "Technical":{ "cache_expire_positive":(tp.Integer(), "Image Cache Expiration", 60, "Days until images are refetched"), "cache_expire_negative":(tp.Integer(), "Image Cache Negative Expiration", 5, "Days until failed image fetches are reattempted"), - "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 50, "RAM Usage in percent at which Maloja should no longer increase its database cache."), + "db_max_memory":(tp.Integer(min=0,max=100), "RAM Percentage soft limit", 70, "RAM Usage in percent at which Maloja should no longer increase its database cache."), "use_request_cache":(tp.Boolean(), "Use request-local DB Cache", False), - "use_global_cache":(tp.Boolean(), "Use global DB Cache", True) + "use_global_cache":(tp.Boolean(), "Use global DB Cache", True, "This is vital for Maloja's performance. Do not disable this unless you have a strong reason to.") }, "Fluff":{ - "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold", 250, "How many scrobbles a track needs to be considered 'Gold' status"), - "scrobbles_platinum":(tp.Integer(), "Scrobbles for Platinum", 500, "How many scrobbles a track needs to be considered 'Platinum' status"), - "scrobbles_diamond":(tp.Integer(), "Scrobbles for Diamond", 1000, "How many scrobbles a track needs to be considered 'Diamond' status"), + "scrobbles_gold":(tp.Integer(), "Scrobbles for Gold (Track)", 250, "How many scrobbles a track needs to be considered 'Gold' status"), + "scrobbles_platinum":(tp.Integer(), "Scrobbles for Platinum (Track)",500, "How many scrobbles a track needs to be considered 'Platinum' status"), + "scrobbles_diamond":(tp.Integer(), "Scrobbles for Diamond (Track)",1000, "How many scrobbles a track needs to be considered 'Diamond' status"), + "scrobbles_gold_album":(tp.Integer(), "Scrobbles for Gold (Album)", 500, "How many scrobbles an album needs to be considered 'Gold' status"), + "scrobbles_platinum_album":(tp.Integer(), "Scrobbles for Platinum (Album)",750, "How many scrobbles an album needs to be considered 'Platinum' status"), + "scrobbles_diamond_album":(tp.Integer(), "Scrobbles for Diamond (Album)",1500, "How many scrobbles an album needs to be considered 'Diamond' status"), "name":(tp.String(), "Name", "Generic Maloja User") }, "Third Party Services":{ @@ -177,6 +182,7 @@ malojaconfig = Configuration( }, "Database":{ + "album_information_trust":(tp.Choice({'first':"First",'last':"Last",'majority':"Majority"}), "Album Information Authority","first", "Whether to trust the first album information that is sent with a track or update every time a different album is sent"), "invalid_artists":(tp.Set(tp.String()), "Invalid Artists", ["[Unknown Artist]","Unknown Artist","Spotify"], "Artists that should be discarded immediately"), "remove_from_title":(tp.Set(tp.String()), "Remove from Title", ["(Original Mix)","(Radio Edit)","(Album Version)","(Explicit Version)","(Bonus Track)"], "Phrases that should be removed from song titles"), "delimiters_feat":(tp.Set(tp.String()), "Featuring Delimiters", ["ft.","ft","feat.","feat","featuring"], "Delimiters used for extra artists, even when in the title field"), @@ -186,11 +192,14 @@ malojaconfig = Configuration( "parse_remix_artists":(tp.Boolean(), "Parse Remix Artists", False) }, "Web Interface":{ - "default_range_charts_artists":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}), "Default Range Artist Charts", "year"), - "default_range_charts_tracks":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}), "Default Range Track Charts", "year"), + "default_range_startpage":(tp.Choice({'alltime':'All Time','year':'Year','month':"Month",'week':'Week'}), "Default Range for Startpage Stats", "year"), "default_step_pulse":(tp.Choice({'year':'Year','month':"Month",'week':'Week','day':'Day'}), "Default Pulse Step", "month"), "charts_display_tiles":(tp.Boolean(), "Display Chart Tiles", False), + "album_showcase":(tp.Boolean(), "Display Album Showcase", True, "Display a graphical album showcase for artist overview pages instead of a chart list"), "display_art_icons":(tp.Boolean(), "Display Album/Artist Icons", True), + "default_album_artist":(tp.String(), "Default Albumartist", "Various Artists"), + "use_album_artwork_for_tracks":(tp.Boolean(), "Use Album Artwork for tracks", True), + "fancy_placeholder_art":(tp.Boolean(), "Use fancy placeholder artwork",True), "discourage_cpu_heavy_stats":(tp.Boolean(), "Discourage CPU-heavy stats", False, "Prevent visitors from mindlessly clicking on CPU-heavy options. Does not actually disable them for malicious actors!"), "use_local_images":(tp.Boolean(), "Use Local Images", True), #"local_image_rotate":(tp.Integer(), "Local Image Rotate", 3600), @@ -297,15 +306,6 @@ data_dir = { -### write down the last ran version -with open(pthj(dir_settings['state'],".lastmalojaversion"),"w") as filed: - filed.write(VERSION) - filed.write("\n") - - - - - ### DOREAH CONFIGURATION from doreah import config @@ -331,7 +331,8 @@ config( custom_css_files = [f for f in os.listdir(data_dir['css']()) if f.lower().endswith('.css')] - +from ..database.sqldb import set_maloja_info +set_maloja_info({'last_run_version':VERSION}) # what the fuck did i just write # this spaghetti file is proudly sponsored by the rice crackers i'm eating at the diff --git a/maloja/proccontrol/tasks/__init__.py b/maloja/proccontrol/tasks/__init__.py index cf2cd85..90ce051 100644 --- a/maloja/proccontrol/tasks/__init__.py +++ b/maloja/proccontrol/tasks/__init__.py @@ -1,3 +1,4 @@ from .import_scrobbles import import_scrobbles from .backup import backup from .export import export # read that line out loud +from .parse_albums import parse_albums diff --git a/maloja/proccontrol/tasks/import_scrobbles.py b/maloja/proccontrol/tasks/import_scrobbles.py index 5c583f5..3d06ad3 100644 --- a/maloja/proccontrol/tasks/import_scrobbles.py +++ b/maloja/proccontrol/tasks/import_scrobbles.py @@ -49,7 +49,7 @@ def import_scrobbles(inputf): typeid,typedesc = "spotify","Spotify" importfunc = parse_spotify_lite_legacy - elif re.match(r"maloja_export_[0-9]+\.json",filename): + elif re.match(r"maloja_export[_0-9]*\.json",filename): typeid,typedesc = "maloja","Maloja" importfunc = parse_maloja @@ -88,10 +88,6 @@ def import_scrobbles(inputf): # extra info extrainfo = {} - if scrobble.get('album_name'): extrainfo['album_name'] = scrobble['album_name'] - if scrobble.get('album_artist'): extrainfo['album_artist'] = scrobble['album_artist'] - # saving this in the scrobble instead of the track because for now it's not meant - # to be authorative information, just payload of the scrobble scrobblebuffer.append({ "time":scrobble['scrobble_time'], @@ -99,6 +95,11 @@ def import_scrobbles(inputf): "artists":scrobble['track_artists'], "title":scrobble['track_title'], "length":scrobble['track_length'], + "album":{ + "albumtitle":scrobble.get('album_name') or None, + "artists":scrobble.get('album_artists') or scrobble['track_artists'] or None + # TODO: use same heuristics as with parsing to determine album? + } if scrobble.get('album_name') else None }, "duration":scrobble['scrobble_duration'], "origin":"import:" + typeid, @@ -441,7 +442,8 @@ def parse_maloja(inputf): 'track_title': s['track']['title'], 'track_artists': s['track']['artists'], 'track_length': s['track']['length'], - 'album_name': s['track'].get('album',{}).get('name',''), + 'album_name': s['track'].get('album',{}).get('albumtitle','') if s['track'].get('album') is not None else '', + 'album_artists': s['track'].get('album',{}).get('artists',None) if s['track'].get('album') is not None else '', 'scrobble_time': s['time'], 'scrobble_duration': s['duration'] },'') diff --git a/maloja/proccontrol/tasks/parse_albums.py b/maloja/proccontrol/tasks/parse_albums.py new file mode 100644 index 0000000..e499818 --- /dev/null +++ b/maloja/proccontrol/tasks/parse_albums.py @@ -0,0 +1,108 @@ +from doreah.io import col + +def parse_albums(strategy=None,prefer_existing=False): + + if strategy not in ("track","none","all","majority","most"): + print(""" +Please specify your album parsing strategy: + + --strategy Specify what strategy to use when the scrobble contains + no information about album artists. + track Take the track artists. This can lead to + separate albums being created for compilation + albums or albums that have collaboration tracks. + none Merge all albums with the same name and assign + 'Various Artists' as the album artist. + all Merge all albums with the same name and assign + every artist that appears on the album as an album + artist. + majority Merge all albums with the same name and assign + artists that appear in at least half the tracks + of the album as album artists. [RECOMMENDED] + most Merge all albums with the same name and assign + the artist that appears most on the album as album + artist. + --prefer_existing If an album with the same name already exists, use it + without further examination of track artists. + """) + return + + + + from ...database.sqldb import guess_albums, get_album_id, add_track_to_album + + print("Parsing album information...") + result = guess_albums() + + result = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]} + print("Found",col['yellow'](len(result)),"Tracks to assign albums to") + + result_authorative = {track_id:result[track_id] for track_id in result if result[track_id]["assigned"]["artists"]} + result_guesswork = {track_id:result[track_id] for track_id in result if not result[track_id]["assigned"]["artists"]} + + i = 0 + + def countup(i): + i+=1 + if (i % 100) == 0: + print(f"Added album information for {i} of {len(result)} tracks...") + return i + + for track_id in result_authorative: + albuminfo = result[track_id]['assigned'] + album_id = get_album_id(albuminfo) + add_track_to_album(track_id,album_id) + i=countup(i) + + albums = {} + for track_id in result_guesswork: + albuminfo = result[track_id]['assigned'] + + # check if already exists + if prefer_existing: + album_id = get_album_id(albuminfo,ignore_albumartists=True,create_new=False) + if album_id: + add_track_to_album(track_id,album_id) + i=countup(i) + continue + + if strategy == 'track': + albuminfo['artists'] = result[track_id]['guess_artists'] + album_id = get_album_id(albuminfo) + add_track_to_album(track_id,album_id) + i=countup(i) + continue + + if strategy == 'none': + albuminfo['artists'] = [] + album_id = get_album_id(albuminfo) + add_track_to_album(track_id,album_id) + i=countup(i) + continue + + if strategy in ['all','majority','most']: + cleantitle = albuminfo['albumtitle'].lower() + albums.setdefault(cleantitle,{'track_ids':[],'artists':{},'title':albuminfo['albumtitle']}) + albums[cleantitle]['track_ids'].append(track_id) + for a in result[track_id]['guess_artists']: + albums[cleantitle]['artists'].setdefault(a,0) + albums[cleantitle]['artists'][a] += 1 + + + for cleantitle in albums: + artistoptions = albums[cleantitle]['artists'] + track_ids = albums[cleantitle]['track_ids'] + realtitle = albums[cleantitle]['title'] + if strategy == 'all': + artists = [a for a in artistoptions] + elif strategy == 'majority': + artists = [a for a in artistoptions if artistoptions[a] >= (len(track_ids) / 2)] + elif strategy == 'most': + artists = [max(artistoptions,key=artistoptions.get)] + + for track_id in track_ids: + album_id = get_album_id({'albumtitle':realtitle,'artists':artists}) + add_track_to_album(track_id,album_id) + i=countup(i) + + print(col['lawngreen']("Done!")) diff --git a/maloja/server.py b/maloja/server.py index ce3595c..a2ff5bc 100644 --- a/maloja/server.py +++ b/maloja/server.py @@ -1,9 +1,7 @@ # technical -import sys import os from threading import Thread from importlib import resources -import datauri import time @@ -19,9 +17,10 @@ from doreah import auth # rest of the project from . import database from .database.jinjaview import JinjaDBConnection -from .images import resolve_track_image, resolve_artist_image +from .images import image_request from .malojauri import uri_to_internal, remove_identical from .pkg_global.conf import malojaconfig, data_dir +from .pkg_global import conf from .jinjaenv.context import jinja_environment from .apis import init_apis, apikeystore @@ -120,20 +119,14 @@ def deprecated_api(pth): @webserver.route("/image") def dynamic_image(): keys = FormsDict.decode(request.query) - if keys['type'] == 'track': - result = resolve_track_image(keys['id']) - elif keys['type'] == 'artist': - result = resolve_artist_image(keys['id']) + result = image_request(**{k:int(keys[k]) for k in keys}) - if result is None or result['value'] in [None,'']: - return "" - if result['type'] == 'raw': - # data uris are directly served as image because a redirect to a data uri - # doesnt work - duri = datauri.DataURI(result['value']) - response.content_type = duri.mimetype - return duri.data - if result['type'] == 'url': + if result['type'] == 'noimage' and result['value'] == 'wait': + # still being worked on + response.status = 202 + response.set_header('Retry-After',15) + return + if result['type'] in ('url','localurl'): redirect(result['value'],307) @webserver.route("/images/") @@ -160,6 +153,9 @@ def static_image(pth): resp.set_header("Content-Type", "image/" + ext) return resp +@webserver.route("/cacheimages/") +def static_proxied_image(uuid): + return static_file(uuid,root=data_dir['cache']('images')) @webserver.route("/login") def login(): @@ -218,8 +214,8 @@ def jinja_page(name): res = template.render(**loc_context) except TemplateNotFound: abort(404,f"Not found: '{name}'") - except (ValueError, IndexError): - abort(404,"This Artist or Track does not exist") + #except (ValueError, IndexError): + # abort(404,"This Artist or Track does not exist") if malojaconfig["DEV_MODE"]: jinja_environment.cache.clear() @@ -283,6 +279,8 @@ logging.getLogger().addHandler(WaitressLogHandler()) def run_server(): + conf.AUX_MODE = False + log("Starting up Maloja server...") ## start database diff --git a/maloja/setup.py b/maloja/setup.py index b74dbab..59c7c7e 100644 --- a/maloja/setup.py +++ b/maloja/setup.py @@ -1,8 +1,10 @@ import os from importlib import resources -from distutils import dir_util - +try: + from setuptools import distutils +except ImportError: + import distutils from doreah.io import col, ask, prompt from doreah import auth @@ -23,7 +25,7 @@ ext_apikeys = [ def copy_initial_local_files(): with resources.files("maloja") / 'data_files' as folder: for cat in dir_settings: - dir_util.copy_tree(os.path.join(folder,cat),dir_settings[cat],update=False) + distutils.dir_util.copy_tree(os.path.join(folder,cat),dir_settings[cat],update=False) charset = list(range(10)) + list("abcdefghijklmnopqrstuvwxyz") + list("ABCDEFGHIJKLMNOPQRSTUVWXYZ") def randomstring(length=32): diff --git a/maloja/thirdparty/__init__.py b/maloja/thirdparty/__init__.py index 135d792..7c582d0 100644 --- a/maloja/thirdparty/__init__.py +++ b/maloja/thirdparty/__init__.py @@ -10,6 +10,7 @@ import xml.etree.ElementTree as ElementTree import json import urllib.parse, urllib.request import base64 +import time from doreah.logging import log from threading import BoundedSemaphore @@ -23,6 +24,14 @@ services = { "metadata":[] } + + +class InvalidResponse(Exception): + """Invalid Response from Third Party""" + +class RateLimitExceeded(Exception): + """Rate Limit exceeded""" + # have a limited number of worker threads so we don't completely hog the cpu with # these requests. they are mostly network bound, so python will happily open up 200 new # requests and then when all the responses come in we suddenly can't load pages anymore @@ -44,26 +53,37 @@ def get_image_track_all(track): for service in services["metadata"]: try: res = service.get_image_track(track) - if res is not None: + if res: log("Got track image for " + str(track) + " from " + service.name) return res else: - log("Could not get track image for " + str(track) + " from " + service.name) + log(f"Could not get track image for {track} from {service.name}") except Exception as e: - log("Error getting track image from " + service.name + ": " + repr(e)) + log(f"Error getting track image from {service.name}: {e.__doc__}") def get_image_artist_all(artist): with thirdpartylock: for service in services["metadata"]: try: res = service.get_image_artist(artist) - if res is not None: + if res: log("Got artist image for " + str(artist) + " from " + service.name) return res else: - log("Could not get artist image for " + str(artist) + " from " + service.name) + log(f"Could not get artist image for {artist} from {service.name}") except Exception as e: - log("Error getting artist image from " + service.name + ": " + repr(e)) - + log(f"Error getting artist image from {service.name}: {e.__doc__}") +def get_image_album_all(album): + with thirdpartylock: + for service in services["metadata"]: + try: + res = service.get_image_album(album) + if res: + log("Got album image for " + str(album) + " from " + service.name) + return res + else: + log(f"Could not get album image for {album} from {service.name}") + except Exception as e: + log(f"Error getting album image from {service.name}: {e.__doc__}") class GenericInterface: @@ -177,6 +197,8 @@ class MetadataInterface(GenericInterface,abstract=True): "activated_setting":None } + delay = 0 + # service provides this role only if the setting is active AND all # necessary auth settings exist def active_metadata(self): @@ -200,6 +222,7 @@ class MetadataInterface(GenericInterface,abstract=True): else: imgurl = None if imgurl is not None: imgurl = self.postprocess_url(imgurl) + time.sleep(self.delay) return imgurl def get_image_artist(self,artist): @@ -215,6 +238,25 @@ class MetadataInterface(GenericInterface,abstract=True): else: imgurl = None if imgurl is not None: imgurl = self.postprocess_url(imgurl) + time.sleep(self.delay) + return imgurl + + def get_image_album(self,album): + artists, title = album + artiststring = urllib.parse.quote(", ".join(artists or [])) + titlestring = urllib.parse.quote(title) + response = urllib.request.urlopen( + self.metadata["albumurl"].format(artist=artiststring,title=titlestring,**self.settings) + ) + + responsedata = response.read() + if self.metadata["response_type"] == "json": + data = json.loads(responsedata) + imgurl = self.metadata_parse_response_album(data) + else: + imgurl = None + if imgurl is not None: imgurl = self.postprocess_url(imgurl) + time.sleep(self.delay) return imgurl # default function to parse response by descending down nodes @@ -225,19 +267,30 @@ class MetadataInterface(GenericInterface,abstract=True): def metadata_parse_response_track(self,data): return self._parse_response("response_parse_tree_track", data) + def metadata_parse_response_album(self,data): + return self._parse_response("response_parse_tree_album", data) + def _parse_response(self, resp, data): res = data for node in self.metadata[resp]: try: res = res[node] except Exception: - return None + handleresult = self.handle_json_result_error(data) #allow the handler to throw custom exceptions + # it can also return True to indicate that this is not an error, but simply an instance of 'this api doesnt have any info' + if handleresult is True: + return None + #throw the generic error if the handler refused to do anything + raise InvalidResponse() return res def postprocess_url(self,url): url = url.replace("http:","https:",1) return url + def handle_json_result_error(self,result): + raise InvalidResponse() + diff --git a/maloja/thirdparty/audiodb.py b/maloja/thirdparty/audiodb.py index 66d2b84..9a4d84a 100644 --- a/maloja/thirdparty/audiodb.py +++ b/maloja/thirdparty/audiodb.py @@ -9,13 +9,17 @@ class AudioDB(MetadataInterface): } metadata = { - #"trackurl": "https://theaudiodb.com/api/v1/json/{api_key}/searchtrack.php?s={artist}&t={title}", + #"trackurl": "https://theaudiodb.com/api/v1/json/{api_key}/searchtrack.php?s={artist}&t={title}", #patreon "artisturl": "https://www.theaudiodb.com/api/v1/json/{api_key}/search.php?s={artist}", + #"albumurl": "https://www.theaudiodb.com/api/v1/json/{api_key}/searchalbum.php?s={artist}&a={title}", #patreon "response_type":"json", #"response_parse_tree_track": ["tracks",0,"astrArtistThumb"], "response_parse_tree_artist": ["artists",0,"strArtistThumb"], "required_settings": ["api_key"], } - def get_image_track(self,artist): + def get_image_track(self,track): + return None + + def get_image_album(self,album): return None diff --git a/maloja/thirdparty/deezer.py b/maloja/thirdparty/deezer.py index 1347c6f..dfac7a8 100644 --- a/maloja/thirdparty/deezer.py +++ b/maloja/thirdparty/deezer.py @@ -1,4 +1,5 @@ -from . import MetadataInterface +from . import MetadataInterface, RateLimitExceeded + class Deezer(MetadataInterface): name = "Deezer" @@ -8,10 +9,31 @@ class Deezer(MetadataInterface): } metadata = { - "trackurl": "https://api.deezer.com/search?q={artist}%20{title}", + #"trackurl": "https://api.deezer.com/search?q={artist}%20{title}", "artisturl": "https://api.deezer.com/search?q={artist}", + "albumurl": "https://api.deezer.com/search?q={artist}%20{title}", "response_type":"json", - "response_parse_tree_track": ["data",0,"album","cover_medium"], + #"response_parse_tree_track": ["data",0,"album","cover_medium"], "response_parse_tree_artist": ["data",0,"artist","picture_medium"], + "response_parse_tree_album": ["data",0,"album","cover_medium"], "required_settings": [], } + + delay = 1 + + def get_image_track(self,track): + return None + # we can use the album pic from the track search, + # but should do so via maloja logic + + + def handle_json_result_error(self,result): + if result.get('data') == []: + return True + if result.get('error',{}).get('code',None) == 4: + self.delay += 1 + # this is permanent (for the lifetime of the process) + # but that's actually ok + # since hitting the rate limit means we are doing this too fast + # and these requests arent really time sensitive + raise RateLimitExceeded() diff --git a/maloja/thirdparty/lastfm.py b/maloja/thirdparty/lastfm.py index 80f3c75..e565e5f 100644 --- a/maloja/thirdparty/lastfm.py +++ b/maloja/thirdparty/lastfm.py @@ -22,15 +22,22 @@ class LastFM(MetadataInterface, ProxyScrobbleInterface): "activated_setting": "SCROBBLE_LASTFM" } metadata = { + #"artisturl": "https://ws.audioscrobbler.com/2.0/?method=artist.getinfo&artist={artist}&api_key={apikey}&format=json" "trackurl": "https://ws.audioscrobbler.com/2.0/?method=track.getinfo&track={title}&artist={artist}&api_key={apikey}&format=json", + "albumurl": "https://ws.audioscrobbler.com/2.0/?method=album.getinfo&api_key={apikey}&artist={artist}&album={title}&format=json", "response_type":"json", "response_parse_tree_track": ["track","album","image",-1,"#text"], + # technically just the album artwork, but we use it for now + #"response_parse_tree_artist": ["artist","image",-1,"#text"], + "response_parse_tree_album": ["album","image",-1,"#text"], "required_settings": ["apikey"], } def get_image_artist(self,artist): return None - # lastfm doesn't provide artist images + # lastfm still provides that endpoint with data, + # but doesn't provide actual images + def proxyscrobble_parse_response(self,data): return data.attrib.get("status") == "ok" and data.find("scrobbles").attrib.get("ignored") == "0" diff --git a/maloja/thirdparty/musicbrainz.py b/maloja/thirdparty/musicbrainz.py index 78e033a..f16229b 100644 --- a/maloja/thirdparty/musicbrainz.py +++ b/maloja/thirdparty/musicbrainz.py @@ -26,6 +26,8 @@ class MusicBrainz(MetadataInterface): return None # not supported + def get_image_album(self,album): + return None def get_image_track(self,track): self.lock.acquire() diff --git a/maloja/thirdparty/spotify.py b/maloja/thirdparty/spotify.py index 8d50284..3a8e55f 100644 --- a/maloja/thirdparty/spotify.py +++ b/maloja/thirdparty/spotify.py @@ -14,10 +14,12 @@ class Spotify(MetadataInterface): } metadata = { - "trackurl": "https://api.spotify.com/v1/search?q=artist:{artist}%20track:{title}&type=track&access_token={token}", - "artisturl": "https://api.spotify.com/v1/search?q=artist:{artist}&type=artist&access_token={token}", + "trackurl": "https://api.spotify.com/v1/search?q={title}%20artist:{artist}&type=track&access_token={token}", + "albumurl": "https://api.spotify.com/v1/search?q={title}%20artist:{artist}&type=album&access_token={token}", + "artisturl": "https://api.spotify.com/v1/search?q={artist}&type=artist&access_token={token}", "response_type":"json", - "response_parse_tree_track": ["tracks","items",0,"album","images",0,"url"], + "response_parse_tree_track": ["tracks","items",0,"album","images",0,"url"], # use album art + "response_parse_tree_album": ["albums","items",0,"images",0,"url"], "response_parse_tree_artist": ["artists","items",0,"images",0,"url"], "required_settings": ["apiid","secret"], } @@ -44,7 +46,7 @@ class Spotify(MetadataInterface): else: expire = responsedata.get("expires_in",3600) self.settings["token"] = responsedata["access_token"] - log("Successfully authenticated with Spotify") + #log("Successfully authenticated with Spotify") Timer(expire,self.authorize).start() except Exception as e: log("Error while authenticating with Spotify: " + repr(e)) diff --git a/maloja/upgrade.py b/maloja/upgrade.py index 67d1176..a4f20cc 100644 --- a/maloja/upgrade.py +++ b/maloja/upgrade.py @@ -11,6 +11,9 @@ from .pkg_global.conf import data_dir, dir_settings from .apis import _apikeys +from .database.sqldb import get_maloja_info, set_maloja_info + + # Dealing with old style tsv files - these should be phased out everywhere def read_tsvs(path,types): result = [] @@ -40,7 +43,7 @@ def upgrade_apikeys(): except Exception: pass - +# v2 to v3 iupgrade def upgrade_db(callback_add_scrobbles): oldfolder = os.path.join(dir_settings['state'],"scrobbles") @@ -88,3 +91,13 @@ def upgrade_db(callback_add_scrobbles): callback_add_scrobbles(scrobblelist) os.rename(os.path.join(oldfolder,sf),os.path.join(newfolder,sf)) log("Done!",color='yellow') + + +# 3.2 album support +def parse_old_albums(): + setting_name = "db_upgrade_albums" + if get_maloja_info([setting_name]).get(setting_name): + pass + else: + pass + #set_maloja_info({setting_name:True}) diff --git a/maloja/web/jinja/abstracts/admin.jinja b/maloja/web/jinja/abstracts/admin.jinja index ff14fd5..12a7b73 100644 --- a/maloja/web/jinja/abstracts/admin.jinja +++ b/maloja/web/jinja/abstracts/admin.jinja @@ -21,7 +21,8 @@ ['setup','Server Setup'], ['settings','Settings'], ['apikeys','API Keys'], - ['manual','Manual Scrobbling'] + ['manual','Manual Scrobbling'], + ['albumless','Tracks without Albums'] ] %} {# ['import','Scrobble Import'], diff --git a/maloja/web/jinja/abstracts/base.jinja b/maloja/web/jinja/abstracts/base.jinja index 3ca1619..be9c746 100644 --- a/maloja/web/jinja/abstracts/base.jinja +++ b/maloja/web/jinja/abstracts/base.jinja @@ -23,13 +23,24 @@ + + {% block scripts %}{% endblock %} - + {% block content %} @@ -56,22 +67,17 @@ +