diff --git a/poetry.lock b/poetry.lock index 626f13b..e665e1e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -9,6 +9,23 @@ python-versions = ">=3.6" [package.dependencies] vine = ">=5.0.0" +[[package]] +name = "anyio" +version = "3.6.2" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +category = "main" +optional = false +python-versions = ">=3.6.2" + +[package.dependencies] +idna = ">=2.8" +sniffio = ">=1.1" + +[package.extras] +doc = ["packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx-rtd-theme"] +test = ["contextlib2", "coverage[toml] (>=4.5)", "hypothesis (>=4.0)", "mock (>=4)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "uvloop (<0.15)", "uvloop (>=0.15)"] +trio = ["trio (>=0.16,<0.22)"] + [[package]] name = "asgiref" version = "3.6.0" @@ -397,6 +414,18 @@ python-versions = "*" [package.dependencies] celery = ">=5.2.3,<6.0" +[[package]] +name = "django-encrypted-field" +version = "1.0.5" +description = "This is a Django Model Field class that can be encrypted using ChaCha20 poly 1305, and other algorithms." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +Django = ">=4.0" +pycryptodomex = ">=3.12.0" + [[package]] name = "django-extensions" version = "3.2.1" @@ -565,10 +594,48 @@ tornado = ["tornado (>=0.2)"] name = "h11" version = "0.14.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "httpcore" +version = "0.16.3" +description = "A minimal low-level HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +anyio = ">=3.0,<5.0" +certifi = "*" +h11 = ">=0.13,<0.15" +sniffio = ">=1.0.0,<2.0.0" + +[package.extras] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] + +[[package]] +name = "httpx" +version = "0.23.3" +description = "The next generation HTTP client." +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +certifi = "*" +httpcore = ">=0.15.0,<0.17.0" +rfc3986 = {version = ">=1.3,<2", extras = ["idna2008"]} +sniffio = "*" + +[package.extras] +brotli = ["brotli", "brotlicffi"] +cli = ["click (>=8.0.0,<9.0.0)", "pygments (>=2.0.0,<3.0.0)", "rich (>=10,<13)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (>=1.0.0,<2.0.0)"] + [[package]] name = "idna" version = "3.4" @@ -853,6 +920,14 @@ category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "pycryptodomex" +version = "3.17" +description = "Cryptographic library for Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + [[package]] name = "pyflakes" version = "2.5.0" @@ -878,6 +953,20 @@ dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pyte docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] +[[package]] +name = "pylast" +version = "5.1.0" +description = "A Python interface to Last.fm and Libre.fm" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +httpx = "*" + +[package.extras] +tests = ["flaky", "pytest", "pytest-cov", "pytest-random-order", "pyyaml"] + [[package]] name = "pysocks" version = "1.7.1" @@ -1178,6 +1267,20 @@ requests = ">=2.0.0" [package.extras] rsa = ["oauthlib[signedtoken] (>=3.0.0)"] +[[package]] +name = "rfc3986" +version = "1.5.0" +description = "Validating URI References per RFC 3986" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +idna = {version = "*", optional = true, markers = "extra == \"idna2008\""} + +[package.extras] +idna2008 = ["idna"] + [[package]] name = "selenium" version = "4.7.2" @@ -1225,7 +1328,7 @@ python-versions = ">=3.6" name = "sniffio" version = "1.3.0" description = "Sniff out which async library your code is running under" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" @@ -1506,13 +1609,17 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "0e23dbecb64cbef4dfe51bdf47e0f6b1357aab1d34342fef5341eaead2c26f1e" +content-hash = "4dd19487f8d9467b920e75a7b3e0c5ae0c735e27b39695d4445537b7524f9065" [metadata.files] amqp = [ {file = "amqp-5.1.1-py3-none-any.whl", hash = "sha256:6f0956d2c23d8fa6e7691934d8c3930eadb44972cbbd1a7ae3a520f735d43359"}, {file = "amqp-5.1.1.tar.gz", hash = "sha256:2c1b13fecc0893e946c65cbd5f36427861cffa4ea2201d8f6fca22e2a373b5e2"}, ] +anyio = [ + {file = "anyio-3.6.2-py3-none-any.whl", hash = "sha256:fbbe32bd270d2a2ef3ed1c5d45041250284e31fc0a4df4a5a6071842051a51e3"}, + {file = "anyio-3.6.2.tar.gz", hash = "sha256:25ea0d673ae30af41a0c442f81cf3b38c7e79fdc7b60335a4c14e05eb0947421"}, +] asgiref = [ {file = "asgiref-3.6.0-py3-none-any.whl", hash = "sha256:71e68008da809b957b7ee4b43dbccff33d1b23519fb8344e33f049897077afac"}, {file = "asgiref-3.6.0.tar.gz", hash = "sha256:9567dfe7bd8d3c8c892227827c41cce860b368104c3431da67a0c5a65a949506"}, @@ -1866,6 +1973,9 @@ django-celery-results = [ {file = "django_celery_results-2.4.0-py3-none-any.whl", hash = "sha256:be91307c02fbbf0dda21993c3001c60edb74595444ccd6ad696552fe3689e85b"}, {file = "django_celery_results-2.4.0.tar.gz", hash = "sha256:75aa51970db5691cbf242c6a0ff50c8cdf419e265cd0e9b772335d06436c4b99"}, ] +django-encrypted-field = [ + {file = "django-encrypted-field-1.0.5.tar.gz", hash = "sha256:e5dbd6d7d1397feb46930b216d7f0806624ebf518bd3fc510b74efb78ee78b6e"}, +] django-extensions = [ {file = "django-extensions-3.2.1.tar.gz", hash = "sha256:2a4f4d757be2563cd1ff7cfdf2e57468f5f931cc88b23cf82ca75717aae504a4"}, {file = "django_extensions-3.2.1-py3-none-any.whl", hash = "sha256:421464be390289513f86cb5e18eb43e5dc1de8b4c27ba9faa3b91261b0d67e09"}, @@ -1984,6 +2094,14 @@ h11 = [ {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, ] +httpcore = [ + {file = "httpcore-0.16.3-py3-none-any.whl", hash = "sha256:da1fb708784a938aa084bde4feb8317056c55037247c787bd7e19eb2c2949dc0"}, + {file = "httpcore-0.16.3.tar.gz", hash = "sha256:c5d6f04e2fc530f39e0c077e6a30caa53f1451096120f1f38b954afd0b17c0cb"}, +] +httpx = [ + {file = "httpx-0.23.3-py3-none-any.whl", hash = "sha256:a211fcce9b1254ea24f0cd6af9869b3d29aba40154e947d2a07bb499b3e310d6"}, + {file = "httpx-0.23.3.tar.gz", hash = "sha256:9818458eb565bb54898ccb9b8b251a28785dd4a55afbc23d0eb410754fe7d0f9"}, +] idna = [ {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, @@ -2259,6 +2377,41 @@ pycparser = [ {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, ] +pycryptodomex = [ + {file = "pycryptodomex-3.17-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:12056c38e49d972f9c553a3d598425f8a1c1d35b2e4330f89d5ff1ffb70de041"}, + {file = "pycryptodomex-3.17-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab33c2d9f275e05e235dbca1063753b5346af4a5cac34a51fa0da0d4edfb21d7"}, + {file = "pycryptodomex-3.17-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:caa937ff29d07a665dfcfd7a84f0d4207b2ebf483362fa9054041d67fdfacc20"}, + {file = "pycryptodomex-3.17-cp27-cp27m-manylinux2014_aarch64.whl", hash = "sha256:db23d7341e21b273d2440ec6faf6c8b1ca95c8894da612e165be0b89a8688340"}, + {file = "pycryptodomex-3.17-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:f854c8476512cebe6a8681cc4789e4fcff6019c17baa0fd72b459155dc605ab4"}, + {file = "pycryptodomex-3.17-cp27-cp27m-win32.whl", hash = "sha256:a57e3257bacd719769110f1f70dd901c5b6955e9596ad403af11a3e6e7e3311c"}, + {file = "pycryptodomex-3.17-cp27-cp27m-win_amd64.whl", hash = "sha256:d38ab9e53b1c09608ba2d9b8b888f1e75d6f66e2787e437adb1fecbffec6b112"}, + {file = "pycryptodomex-3.17-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:3c2516b42437ae6c7a29ef3ddc73c8d4714e7b6df995b76be4695bbe4b3b5cd2"}, + {file = "pycryptodomex-3.17-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5c23482860302d0d9883404eaaa54b0615eefa5274f70529703e2c43cc571827"}, + {file = "pycryptodomex-3.17-cp27-cp27mu-manylinux2014_aarch64.whl", hash = "sha256:7a8dc3ee7a99aae202a4db52de5a08aa4d01831eb403c4d21da04ec2f79810db"}, + {file = "pycryptodomex-3.17-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:7cc28dd33f1f3662d6da28ead4f9891035f63f49d30267d3b41194c8778997c8"}, + {file = "pycryptodomex-3.17-cp35-abi3-macosx_10_9_universal2.whl", hash = "sha256:2d4d395f109faba34067a08de36304e846c791808524614c731431ee048fe70a"}, + {file = "pycryptodomex-3.17-cp35-abi3-macosx_10_9_x86_64.whl", hash = "sha256:55eed98b4150a744920597c81b3965b632038781bab8a08a12ea1d004213c600"}, + {file = "pycryptodomex-3.17-cp35-abi3-manylinux2014_aarch64.whl", hash = "sha256:7fa0b52df90343fafe319257b31d909be1d2e8852277fb0376ba89d26d2921db"}, + {file = "pycryptodomex-3.17-cp35-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78f0ddd4adc64baa39b416f3637aaf99f45acb0bcdc16706f0cc7ebfc6f10109"}, + {file = "pycryptodomex-3.17-cp35-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a4fa037078e92c7cc49f6789a8bac3de06856740bb2038d05f2d9a2e4b165d59"}, + {file = "pycryptodomex-3.17-cp35-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:88b0d5bb87eaf2a31e8a759302b89cf30c97f2f8ca7d83b8c9208abe8acb447a"}, + {file = "pycryptodomex-3.17-cp35-abi3-musllinux_1_1_i686.whl", hash = "sha256:6feedf4b0e36b395329b4186a805f60f900129cdf0170e120ecabbfcb763995d"}, + {file = "pycryptodomex-3.17-cp35-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:7a6651a07f67c28b6e978d63aa3a3fccea0feefed9a8453af3f7421a758461b7"}, + {file = "pycryptodomex-3.17-cp35-abi3-win32.whl", hash = "sha256:32e764322e902bbfac49ca1446604d2839381bbbdd5a57920c9daaf2e0b778df"}, + {file = "pycryptodomex-3.17-cp35-abi3-win_amd64.whl", hash = "sha256:4b51e826f0a04d832eda0790bbd0665d9bfe73e5a4d8ea93b6a9b38beeebe935"}, + {file = "pycryptodomex-3.17-pp27-pypy_73-macosx_10_9_x86_64.whl", hash = "sha256:d4cf0128da167562c49b0e034f09e9cedd733997354f2314837c2fa461c87bb1"}, + {file = "pycryptodomex-3.17-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:c92537b596bd5bffb82f8964cabb9fef1bca8a28a9e0a69ffd3ec92a4a7ad41b"}, + {file = "pycryptodomex-3.17-pp27-pypy_73-win32.whl", hash = "sha256:599bb4ae4bbd614ca05f49bd4e672b7a250b80b13ae1238f05fd0f09d87ed80a"}, + {file = "pycryptodomex-3.17-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:4c4674f4b040321055c596aac926d12f7f6859dfe98cd12f4d9453b43ab6adc8"}, + {file = "pycryptodomex-3.17-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:67a3648025e4ddb72d43addab764336ba2e670c8377dba5dd752e42285440d31"}, + {file = "pycryptodomex-3.17-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40e8a11f578bd0851b02719c862d55d3ee18d906c8b68a9c09f8c564d6bb5b92"}, + {file = "pycryptodomex-3.17-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:23d83b610bd97704f0cd3acc48d99b76a15c8c1540d8665c94d514a49905bad7"}, + {file = "pycryptodomex-3.17-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fd29d35ac80755e5c0a99d96b44fb9abbd7e871849581ea6a4cb826d24267537"}, + {file = "pycryptodomex-3.17-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64b876d57cb894b31056ad8dd6a6ae1099b117ae07a3d39707221133490e5715"}, + {file = "pycryptodomex-3.17-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee8bf4fdcad7d66beb744957db8717afc12d176e3fd9c5d106835133881a049b"}, + {file = "pycryptodomex-3.17-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:c84689c73358dfc23f9fdcff2cb9e7856e65e2ce3b5ed8ff630d4c9bdeb1867b"}, + {file = "pycryptodomex-3.17.tar.gz", hash = "sha256:0af93aad8d62e810247beedef0261c148790c52f3cd33643791cc6396dd217c1"}, +] pyflakes = [ {file = "pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2"}, {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, @@ -2267,6 +2420,10 @@ pyjwt = [ {file = "PyJWT-2.6.0-py3-none-any.whl", hash = "sha256:d83c3d892a77bbb74d3e1a2cfa90afaadb60945205d1095d9221f04466f64c14"}, {file = "PyJWT-2.6.0.tar.gz", hash = "sha256:69285c7e31fc44f68a1feb309e948e0df53259d579295e6cfe2b1792329f05fd"}, ] +pylast = [ + {file = "pylast-5.1.0-py3-none-any.whl", hash = "sha256:73cc7429a57e4965b3769254b1cb9625cdd910d3ac26cb0a1dd57145cdc498c0"}, + {file = "pylast-5.1.0.tar.gz", hash = "sha256:89300fdcdf423d7be0606bdc44da27c3b48c4d73aa1d4cb12672cc006c979bdc"}, +] pysocks = [ {file = "PySocks-1.7.1-py27-none-any.whl", hash = "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299"}, {file = "PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5"}, @@ -2397,6 +2554,10 @@ requests-oauthlib = [ {file = "requests-oauthlib-1.3.1.tar.gz", hash = "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"}, {file = "requests_oauthlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5"}, ] +rfc3986 = [ + {file = "rfc3986-1.5.0-py2.py3-none-any.whl", hash = "sha256:a86d6e1f5b1dc238b218b012df0aa79409667bb209e58da56d0b94704e712a97"}, + {file = "rfc3986-1.5.0.tar.gz", hash = "sha256:270aaf10d87d0d4e095063c65bf3ddbc6ee3d0b226328ce21e036f946e421835"}, +] selenium = [ {file = "selenium-4.7.2-py3-none-any.whl", hash = "sha256:06a1c7d9f313130b21c3218ddd8852070d0e7419afdd31f96160cd576555a5ce"}, {file = "selenium-4.7.2.tar.gz", hash = "sha256:3aefa14a28a42e520550c1cd0f29cf1d566328186ea63aa9a3e01fb265b5894d"}, diff --git a/pyproject.toml b/pyproject.toml index 94f10c4..6fb482b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ pysportsdb = "^0.1.0" django-cachalot = "^2.5.2" pytz = "^2022.7.1" django-redis = "^5.2.0" +pylast = "^5.1.0" +django-encrypted-field = "^1.0.5" [tool.poetry.dev-dependencies] Werkzeug = "2.0.3" diff --git a/vrobbler/apps/music/utils.py b/vrobbler/apps/music/utils.py index a1590e9..ba2bb24 100644 --- a/vrobbler/apps/music/utils.py +++ b/vrobbler/apps/music/utils.py @@ -1,10 +1,12 @@ -#!/usr/bin/env python3 -from typing import Optional +import logging + from scrobbles.musicbrainz import ( lookup_album_dict_from_mb, lookup_artist_id_from_mb, ) +logger = logging.getLogger(__name__) + from music.models import Artist, Album, Track @@ -20,7 +22,7 @@ def get_or_create_artist(name: str) -> Artist: def get_or_create_album(name: str, artist: Artist) -> Album: album = None album_created = False - albums = Album.objects.filter(name=name) + albums = Album.objects.filter(name__iexact=name) if albums.count() == 1: album = albums.first() else: @@ -62,16 +64,24 @@ def get_or_create_track( run_time=None, run_time_ticks=None, ) -> Track: - track, track_created = Track.objects.get_or_create( - title=title, - artist=artist, - musicbrainz_id=mbid, - ) + track = None + if mbid: + track = Track.objects.filter( + musicbrainz_id=mbid, + ).first() + if not track: + track = Track.objects.filter( + title=title, artist=artist, album=album + ).first() - if track_created: - track.album = album - track.run_time = run_time - track.run_time_ticks = run_time_ticks - track.save(update_fields=['album', 'run_time', 'run_time_ticks']) + if not track: + track = Track.objects.create( + title=title, + artist=artist, + album=album, + musicbrainz_id=mbid, + run_time=run_time, + run_time_ticks=run_time_ticks, + ) return track diff --git a/vrobbler/apps/profiles/migrations/0002_userprofile_lastfm_password_and_more.py b/vrobbler/apps/profiles/migrations/0002_userprofile_lastfm_password_and_more.py new file mode 100644 index 0000000..9b01a1c --- /dev/null +++ b/vrobbler/apps/profiles/migrations/0002_userprofile_lastfm_password_and_more.py @@ -0,0 +1,24 @@ +# Generated by Django 4.1.5 on 2023-02-12 22:26 + +from django.db import migrations, models +import encrypted_field.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('profiles', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='userprofile', + name='lastfm_password', + field=encrypted_field.fields.EncryptedField(blank=True, null=True), + ), + migrations.AddField( + model_name='userprofile', + name='lastfm_username', + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vrobbler/apps/profiles/models.py b/vrobbler/apps/profiles/models.py index adc125b..8a540fc 100644 --- a/vrobbler/apps/profiles/models.py +++ b/vrobbler/apps/profiles/models.py @@ -5,7 +5,10 @@ from django.db import models from django_extensions.db.models import TimeStampedModel from profiles.constants import PRETTY_TIMEZONE_CHOICES +from encrypted_field import EncryptedField + User = get_user_model() +BNULL = {"blank": True, "null": True} class UserProfile(TimeStampedModel): @@ -15,6 +18,8 @@ class UserProfile(TimeStampedModel): timezone = models.CharField( max_length=255, choices=PRETTY_TIMEZONE_CHOICES ) + lastfm_username = models.CharField(max_length=255, **BNULL) + lastfm_password = EncryptedField(**BNULL) def __str__(self): return f"User profile for {self.user}" diff --git a/vrobbler/apps/scrobbles/admin.py b/vrobbler/apps/scrobbles/admin.py index f91df68..540bd8f 100644 --- a/vrobbler/apps/scrobbles/admin.py +++ b/vrobbler/apps/scrobbles/admin.py @@ -1,6 +1,10 @@ from django.contrib import admin - -from scrobbles.models import AudioScrobblerTSVImport, Scrobble +from scrobbles.models import ( + AudioScrobblerTSVImport, + ChartRecord, + LastFmImport, + Scrobble, +) class ScrobbleInline(admin.TabularInline): @@ -17,6 +21,38 @@ class AudioScrobblerTSVImportAdmin(admin.ModelAdmin): ordering = ("-created",) +@admin.register(LastFmImport) +class LastFmImportAdmin(admin.ModelAdmin): + date_hierarchy = "created" + list_display = ("uuid", "created", "process_count", "processed_on") + ordering = ("-created",) + + +@admin.register(ChartRecord) +class ChartRecordAdmin(admin.ModelAdmin): + date_hierarchy = "created" + list_display = ( + "user", + "rank", + "year", + "week", + "month", + "day", + "media_name", + ) + ordering = ("-created",) + + def media_name(self, obj): + if obj.video: + return obj.video + if obj.track: + return obj.track + if obj.podcast_episode: + return obj.podcast_episode + if obj.sport_event: + return obj.sport_event + + @admin.register(Scrobble) class ScrobbleAdmin(admin.ModelAdmin): date_hierarchy = "timestamp" diff --git a/vrobbler/apps/scrobbles/lastfm.py b/vrobbler/apps/scrobbles/lastfm.py new file mode 100644 index 0000000..bd93033 --- /dev/null +++ b/vrobbler/apps/scrobbles/lastfm.py @@ -0,0 +1,143 @@ +import logging +import time +from datetime import datetime, timedelta + +import pylast +import pytz +from django.conf import settings +from music.utils import ( + get_or_create_album, + get_or_create_artist, + get_or_create_track, +) + +logger = logging.getLogger(__name__) + +PYLAST_ERRORS = tuple( + getattr(pylast, exc_name) + for exc_name in ( + "ScrobblingError", + "NetworkError", + "MalformedResponseError", + "WSError", + ) + if hasattr(pylast, exc_name) +) + + +class LastFM: + def __init__(self, user): + try: + self.client = pylast.LastFMNetwork( + api_key=getattr(settings, "LASTFM_API_KEY"), + api_secret=getattr(settings, "LASTFM_SECRET_KEY"), + username=user.profile.lastfm_username, + password_hash=pylast.md5(user.profile.lastfm_password), + ) + self.user = self.client.get_user(user.profile.lastfm_username) + self.vrobbler_user = user + except PYLAST_ERRORS as e: + logger.error(f"Error during Last.fm setup: {e}") + + def import_from_lastfm(self, last_processed=None): + """Given a last processed time, import all scrobbles from LastFM since then""" + from scrobbles.models import Scrobble + + new_scrobbles = [] + source = "Last.fm" + source_id = "" + latest_scrobbles = self.get_last_scrobbles(time_from=last_processed) + + for scrobble in latest_scrobbles: + timestamp = scrobble.pop('timestamp') + + artist = get_or_create_artist(scrobble.pop('artist')) + album = get_or_create_album(scrobble.pop('album'), artist) + + scrobble['artist'] = artist + scrobble['album'] = album + track = get_or_create_track(**scrobble) + + new_scrobble = Scrobble( + user=self.vrobbler_user, + timestamp=timestamp, + source=source, + source_id=source_id, + track=track, + played_to_completion=True, + in_progress=False, + ) + # Vrobbler scrobbles on finish, LastFM scrobbles on start + ten_seconds_eariler = timestamp - timedelta(seconds=15) + ten_seconds_later = timestamp + timedelta(seconds=15) + existing = Scrobble.objects.filter( + created__gte=ten_seconds_eariler, + created__lte=ten_seconds_later, + track=track, + ).first() + if existing: + logger.debug(f"Skipping existing scrobble {new_scrobble}") + continue + logger.debug(f"Queued scrobble {new_scrobble} for creation") + new_scrobbles.append(new_scrobble) + + created = Scrobble.objects.bulk_create(new_scrobbles) + logger.info( + f"Created {len(created)} scrobbles", + extra={'created_scrobbles': created}, + ) + return created + + @staticmethod + def undo_lastfm_import(process_log, dryrun=True): + """Given a newline separated list of scrobbles, delete them""" + from scrobbles.models import Scrobble + + if not process_log: + logger.warning("No lines in process log found to undo") + return + + for line in process_log.split('\n'): + scrobble_id = line.split("\t")[0] + scrobble = Scrobble.objects.filter(id=scrobble_id).first() + if not scrobble: + logger.warning( + f"Could not find scrobble {scrobble_id} to undo" + ) + continue + logger.info(f"Removing scrobble {scrobble_id}") + if not dryrun: + scrobble.delete() + + def get_last_scrobbles(self, time_from=None, time_to=None): + """Given a user, Last.fm api key, and secret key, grab a list of scrobbled + tracks""" + scrobbles = [] + if time_from: + time_from = int(time_from.timestamp()) + if time_to: + time_to = int(time_to.timestamp()) + + if not time_from and not time_to: + found_scrobbles = self.user.get_recent_tracks(limit=None) + else: + found_scrobbles = self.user.get_recent_tracks( + time_from=time_from, time_to=time_to + ) + for scrobble in found_scrobbles: + run_time_ticks = scrobble.track.get_duration() + run_time = run_time_ticks / 1000 + scrobbles.append( + { + "artist": scrobble.track.get_artist().name, + "album": scrobble.album, + "title": scrobble.track.title, + "mbid": scrobble.track.get_mbid(), + "run_time": int(run_time), + "run_time_ticks": run_time_ticks, + "timestamp": datetime.utcfromtimestamp( + int(scrobble.timestamp) + ).replace(tzinfo=pytz.utc), + } + ) + return scrobbles diff --git a/vrobbler/apps/scrobbles/migrations/0018_lastfmimport.py b/vrobbler/apps/scrobbles/migrations/0018_lastfmimport.py new file mode 100644 index 0000000..564e662 --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0018_lastfmimport.py @@ -0,0 +1,61 @@ +# Generated by Django 4.1.5 on 2023-02-13 06:43 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('scrobbles', '0017_audioscrobblertsvimport_user'), + ] + + operations = [ + migrations.CreateModel( + name='LastFmImport', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ( + 'created', + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name='created' + ), + ), + ( + 'modified', + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name='modified' + ), + ), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)), + ('processed_on', models.DateTimeField(blank=True, null=True)), + ('process_log', models.TextField(blank=True, null=True)), + ('process_count', models.IntegerField(blank=True, null=True)), + ( + 'user', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + 'get_latest_by': 'modified', + 'abstract': False, + }, + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 300e023..05cfe47 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -12,6 +12,7 @@ from scrobbles.utils import check_scrobble_for_finish from sports.models import SportEvent from videos.models import Series, Video from vrobbler.apps.profiles.utils import now_user_timezone +from vrobbler.apps.scrobbles.lastfm import LastFM logger = logging.getLogger(__name__) User = get_user_model() @@ -78,6 +79,66 @@ class AudioScrobblerTSVImport(TimeStampedModel): undo_audioscrobbler_tsv_import(self.process_log, dryrun) +class LastFmImport(TimeStampedModel): + user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL) + uuid = models.UUIDField(editable=False, default=uuid4) + processed_on = models.DateTimeField(**BNULL) + process_log = models.TextField(**BNULL) + process_count = models.IntegerField(**BNULL) + + def __str__(self): + return f"LastFM Import: {self.uuid}" + + def process(self, import_all=False): + """Import scrobbles found on LastFM""" + if self.processed_on: + logger.info(f"{self} already processed on {self.processed_on}") + return + + last_import = None + if not import_all: + try: + last_import = LastFmImport.objects.exclude(id=self.id).last() + except: + pass + + if not import_all and not last_import: + logger.warn( + "No previous import, to import all Last.fm scrobbles, pass import_all=True" + ) + return + + lastfm = LastFM(self.user) + last_processed = None + if last_import: + last_processed = last_import.processed_on + + scrobbles = lastfm.import_from_lastfm(last_processed) + self.process_log = "" + if scrobbles: + for count, scrobble in enumerate(scrobbles): + scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}" + log_line = f"{scrobble_str}" + if count > 0: + log_line = "\n" + log_line + self.process_log += log_line + self.process_count = len(scrobbles) + else: + self.process_log = f"Created no new scrobbles" + self.process_count = 0 + + self.processed_on = timezone.now() + self.save( + update_fields=['processed_on', 'process_count', 'process_log'] + ) + + def undo(self, dryrun=False): + """Undo import of scrobbles from LastFM""" + LastFM.undo_lastfm_import(self.process_log, dryrun) + self.processed_on = None + self.save(update_fields=['processed_on']) + + class ChartRecord(TimeStampedModel): """Sort of like a materialized view for what we could dynamically generate, but would kill the DB as it gets larger. Collects time-based records diff --git a/vrobbler/settings.py b/vrobbler/settings.py index 98d0a3b..ad19243 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -37,6 +37,12 @@ KEEP_DETAILED_SCROBBLE_LOGS = os.getenv( "VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS", False ) +# Key must be 16, 24 or 32 bytes long and will be converted to a byte stream +ENCRYPTED_FIELD_KEY = os.getenv( + "VROBBLER_ENCRYPTED_FIELD_KEY", "12345678901234567890123456789012" +) + +DJANGO_ENCRYPTED_FIELD_KEY = bytes(ENCRYPTED_FIELD_KEY, "utf-8") # Should we cull old in-progress scrobbles that are beyond the wait period for resuming? DELETE_STALE_SCROBBLES = os.getenv("VROBBLER_DELETE_STALE_SCROBBLES", True) @@ -51,6 +57,9 @@ THESPORTSDB_BASE_URL = os.getenv( ) TMDB_API_KEY = os.getenv("VROBBLER_TMDB_API_KEY", "") +LASTFM_API_KEY = os.getenv("VROBBLER_LASTFM_API_KEY") +LASTFM_SECRET_KEY = os.getenv("VROBBLER_LASTFM_SECRET_KEY") + DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' TIME_ZONE = os.getenv("VROBBLER_TIME_ZONE", "US/Eastern") @@ -75,7 +84,8 @@ INSTALLED_APPS = [ "django.contrib.humanize", "django_filters", "django_extensions", - 'rest_framework.authtoken', + "rest_framework.authtoken", + "encrypted_field", "cachalot", "profiles", "scrobbles",