Bump version to 0.9.3

Issue was not having a user so we couldn't set a timezone. All fixed now

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit=
+  vrobbler/wsgi.py
+  vrobbler/asgi.py
+  vrobbler/cli.py
+  *admin.py
+  migrations/*

.drone.yml

@@ -8,15 +8,15 @@ name: run_tests
 
 steps:
   # Run tests against Python/Flask engine backend (with pytest)
-  - name: django_tests
+  - name: pytest with coverage
     image: python:3.10.4
     commands:
       # Install dependencies
-      - cp vrobbler.conf.example vrobbler.conf
+      - cp vrobbler.conf.test vrobbler.conf
       - pip install poetry
       - poetry install
       # Start with a fresh database (which is already running as a service from Drone)
-      - poetry run python manage.py test
+      - poetry run pytest --cov-report term:skip-covered --cov=vrobbler tests
     environment:
       VROBBLER_DATABASE_URL: sqlite:///test.db
     volumes:

.gitignore

@@ -1,4 +1,5 @@
 db.sqlite3
 vrobbler.conf
-/media/
-/dist/
+media/
+dist/
+.coverage

README.md

@@ -1,4 +1,7 @@
-#+title: Readme
+Vrobbler
+========
+
+[![Build Status](https://ci.unbl.ink/api/badges/secstate/vrobbler/status.svg?ref=refs/heads/main)](https://ci.unbl.ink/secstate/vrobbler)
 
 Vrobbler is a pretty simple Django-powered web app for scrobbling video plays from you favorite Jellyfin installation.
 

envrc.sample

@@ -0,0 +1,4 @@
+export ENV_PATH=$(poetry env info --path)
+source "${ENV_PATH}/bin/activate"
+
+export PYPI_PASSWORD="$(pass personal/apikey/pypi)"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vrobbler"
-version = "0.4.2"
+version = "0.9.3"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 
@@ -16,7 +16,7 @@ djangorestframework = "^3.13.1"
 Markdown = "^3.3.6"
 django-filter = "^21.1"
 Pillow = "^9.0.1"
-psycopg2 = {version = "^2.9.3", extras = ["production"]}
+psycopg2 = "^2.9.3"
 dj-database-url = "^0.5.0"
 django-mathfilters = "^1.0.0"
 django-allauth = "^0.50.0"
@@ -28,24 +28,38 @@ gunicorn = "^20.1.0"
 django-simple-history = "^3.1.1"
 whitenoise = "^6.3.0"
 musicbrainzngs = "^0.7.1"
+cinemagoer = "^2022.12.27"
+pysportsdb = "^0.1.0"
+pytz = "^2022.7.1"
+django-redis = "^5.2.0"
+pylast = "^5.1.0"
+django-encrypted-field = "^1.0.5"
+celery = "^5.2.7"
 
 [tool.poetry.dev-dependencies]
 Werkzeug = "2.0.3"
 black = "^22.3"
-freezegun = "^1.2"
+coverage = "^7.0.5"
 mypy = "^0.961"
 pytest = "^7.1"
 pytest-black = "^0.3.12"
 pytest-cov = "^3.0"
+pytest-django = "^4.5.2"
 pytest-flake8 = "^1.1"
 pytest-isort = "^3.0"
 pytest-runner = "^6.0"
 pytest-selenium = "^2.0.1"
+time-machine = "^2.9.0"
 types-pytz = "^2022.1"
 types-requests = "^2.27"
-types-freezegun = "^1.1"
 bandit = "^1.7.4"
 
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "-ra -q"
+testpaths = ["tests"]
+DJANGO_SETTINGS_MODULE='vrobbler.settings'
+
 [tool.black]
 line-length = 79
 skip-string-normalization = true

tests/scrobbles_tests/conftest.py

@@ -0,0 +1,84 @@
+import json
+import pytest
+
+from scrobbles.models import Scrobble
+from rest_framework.authtoken.models import Token
+from django.contrib.auth import get_user_model
+
+User = get_user_model()
+
+
+class MopidyRequest:
+    name = "Same in the End"
+    artist = "Sublime"
+    album = "Sublime"
+    track_number = 4
+    run_time_ticks = 156604
+    run_time = "156"
+    playback_time_ticks = 15045
+    musicbrainz_track_id = "54214d63-5adf-4909-87cd-c65c37a6d558"
+    musicbrainz_album_id = "03b864cd-7761-314c-a892-05a89ddff00d"
+    musicbrainz_artist_id = "95f5b748-d370-47fe-85bd-0af2dc450bc0"
+    mopidy_uri = "local:track:Sublime%20-%20Sublime/Disc%201%20-%2004%20-%20Same%20in%20the%20End.mp3"
+    status = "resumed"
+
+    def __init__(self, **kwargs):
+        self.request_data = {
+            "name": kwargs.get('name', self.name),
+            "artist": kwargs.get("artist", self.artist),
+            "album": kwargs.get("album", self.album),
+            "track_number": int(kwargs.get("track_number", self.track_number)),
+            "run_time_ticks": int(
+                kwargs.get("run_time_ticks", self.run_time_ticks)
+            ),
+            "run_time": int(kwargs.get("run_time", self.run_time)),
+            "playback_time_ticks": int(
+                kwargs.get("playback_time_ticks", self.playback_time_ticks)
+            ),
+            "musicbrainz_track_id": kwargs.get(
+                "musicbrainz_track_id", self.musicbrainz_track_id
+            ),
+            "musicbrainz_album_id": kwargs.get(
+                "musicbrainz_album_id", self.musicbrainz_album_id
+            ),
+            "musicbrainz_artist_id": kwargs.get(
+                "musicbrainz_artist_id", self.musicbrainz_artist_id
+            ),
+            "mopidy_uri": kwargs.get("mopidy_uri", self.mopidy_uri),
+            "status": kwargs.get("status", self.status),
+        }
+
+    def __eq__(self, other):
+        for key in self.request_data.keys():
+            if self.request_data[key] != getattr(self, key):
+                return False
+        return True
+
+    @property
+    def request_json(self):
+        return json.dumps(self.request_data)
+
+
+@pytest.fixture
+def valid_auth_token():
+    user = User.objects.create(email='test@exmaple.com')
+    return Token.objects.create(user=user).key
+
+
+@pytest.fixture
+def mopidy_track_request_data():
+    return MopidyRequest().request_json
+
+
+@pytest.fixture
+def mopidy_track_diff_album_request_data(**kwargs):
+    mb_album_id = "0c56c457-afe1-4679-baab-759ba8dd2a58"
+    return MopidyRequest(
+        album="Gold", musicbrainz_album_id=mb_album_id
+    ).request_json
+
+
+@pytest.fixture
+def mopidy_podcast_request_data():
+    mopidy_uri = "local:podcast:Up%20First/2022-01-01%20Up%20First.mp3"
+    return MopidyRequest(mopidy_uri=mopidy_uri).request_json

tests/scrobbles_tests/test_aggregators.py

@@ -0,0 +1,107 @@
+from datetime import datetime, timedelta
+
+import pytest
+import time_machine
+from django.contrib.auth import get_user_model
+from django.urls import reverse
+from django.utils import timezone
+from music.aggregators import (
+    scrobble_counts,
+    top_artists,
+    top_tracks,
+    week_of_scrobbles,
+)
+from profiles.models import UserProfile
+from scrobbles.models import Scrobble
+
+
+def build_scrobbles(client, request_data, num=7, spacing=2):
+    url = reverse('scrobbles:mopidy-websocket')
+    user = get_user_model().objects.create(username='Test User')
+    UserProfile.objects.create(user=user, timezone='US/Eastern')
+    for i in range(num):
+        client.post(url, request_data, content_type='application/json')
+        s = Scrobble.objects.last()
+        s.user = user
+        s.timestamp = timezone.now() - timedelta(days=i * spacing)
+        s.played_to_completion = True
+        s.save()
+
+
+@pytest.mark.django_db
+@time_machine.travel(datetime(2022, 3, 4, 1, 24))
+def test_scrobble_counts_data(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data)
+    user = get_user_model().objects.first()
+    count_dict = scrobble_counts(user)
+    assert count_dict == {
+        'alltime': 7,
+        'month': 2,
+        'today': 1,
+        'week': 3,
+        'year': 7,
+    }
+
+
+@pytest.mark.django_db
+def test_week_of_scrobbles_data(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    week = week_of_scrobbles(user)
+    assert list(week.values()) == [1, 1, 1, 1, 1, 1, 1]
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_day(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = top_tracks(user)
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_week(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = top_tracks(user, filter='week')
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_month(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = top_tracks(user, filter='month')
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_year(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = top_tracks(user, filter='year')
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top__artists_by_week(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = top_artists(user, filter='week')
+    assert tops[0].name == "Sublime"
+
+
+@pytest.mark.django_db
+def test_top__artists_by_month(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = top_artists(user, filter='month')
+    assert tops[0].name == "Sublime"
+
+
+@pytest.mark.django_db
+def test_top__artists_by_year(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = top_artists(user, filter='year')
+    assert tops[0].name == "Sublime"

tests/scrobbles_tests/test_imdb.py

@@ -0,0 +1,12 @@
+import pytest
+import imdb
+from mock import patch
+
+from vrobbler.apps.scrobbles.imdb import lookup_video_from_imdb
+
+
+def test_lookup_imdb_bad_id(caplog):
+    data = lookup_video_from_imdb('3409324')
+    assert data is None
+    assert caplog.records[0].levelname == "WARNING"
+    assert caplog.records[0].msg == "IMDB ID should begin with 'tt' 3409324"

tests/scrobbles_tests/test_views.py

@@ -0,0 +1,100 @@
+import json
+import pytest
+
+from django.urls import reverse
+
+from scrobbles.models import Scrobble
+from music.models import Track
+from podcasts.models import Episode
+
+
+@pytest.mark.django_db
+def test_get_not_allowed_from_mopidy(client, valid_auth_token):
+    url = reverse('scrobbles:mopidy-websocket')
+    headers = {'Authorization': f'Token {valid_auth_token}'}
+    response = client.get(url, headers=headers)
+    assert response.status_code == 405
+
+
+@pytest.mark.django_db
+def test_bad_mopidy_request_data(client, valid_auth_token):
+    url = reverse('scrobbles:mopidy-websocket')
+    headers = {'Authorization': f'Token {valid_auth_token}'}
+    response = client.post(url, headers)
+    assert response.status_code == 400
+    assert (
+        response.data['detail']
+        == 'JSON parse error - Expecting value: line 1 column 1 (char 0)'
+    )
+
+
+@pytest.mark.django_db
+def test_scrobble_mopidy_track(
+    client, mopidy_track_request_data, valid_auth_token
+):
+    url = reverse('scrobbles:mopidy-websocket')
+    headers = {'Authorization': f'Token {valid_auth_token}'}
+    response = client.post(
+        url,
+        mopidy_track_request_data,
+        content_type='application/json',
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {'scrobble_id': 1}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.__class__ == Track
+    assert scrobble.media_obj.title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_scrobble_mopidy_same_track_different_album(
+    client,
+    mopidy_track_request_data,
+    mopidy_track_diff_album_request_data,
+    valid_auth_token,
+):
+    url = reverse('scrobbles:mopidy-websocket')
+    headers = {'Authorization': f'Token {valid_auth_token}'}
+    response = client.post(
+        url,
+        mopidy_track_request_data,
+        content_type='application/json',
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {'scrobble_id': 1}
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.album.name == "Sublime"
+
+    response = client.post(
+        url,
+        mopidy_track_diff_album_request_data,
+        content_type='application/json',
+    )
+
+    scrobble = Scrobble.objects.get(id=2)
+    assert scrobble.media_obj.__class__ == Track
+    assert scrobble.media_obj.album.name == "Gold"
+    assert scrobble.media_obj.title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_scrobble_mopidy_podcast(
+    client, mopidy_podcast_request_data, valid_auth_token
+):
+    url = reverse('scrobbles:mopidy-websocket')
+    headers = {'Authorization': f'Token {valid_auth_token}'}
+    response = client.post(
+        url,
+        mopidy_podcast_request_data,
+        content_type='application/json',
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {'scrobble_id': 1}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.__class__ == Episode
+    assert scrobble.media_obj.title == "Up First"

todos.org

@@ -0,0 +1,380 @@
+#+title: TODOs
+
+A fun way to keep track of things in the project to fix or improve.
+
+* DONE [#A] Fix fetching artwork without release group :bug:
+CLOSED: [2023-01-29 Sun 14:27]
+
+When we get artwork from Musicbrianz, and it's not found, we should check for
+release groups as well. This will stop issues with missing artwork because of
+obscure MB release matches.
+
+* DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion percent :bug:
+CLOSED: [2023-01-30 Mon 18:31]
+:LOGBOOK:
+CLOCK: [2023-01-30 Mon 18:00]--[2023-01-30 Mon 18:31] =>  0:31
+:END:
+
+If we play music from Jellyfin and the track reaches 90% completion, the
+scrobbling goes crazy and starts creating new scrobbles with every update.
+
+The cause is pretty simple, but the solution is hard. We want to mark a scrobble
+as complete for the following conditions:
+
+- Play stopped and percent played beyond 90%
+- Play completely finished
+
+But if we keep listening beyond 90, we should basically ignore updates (or just
+update the existing scrobble)
+* DONE [#A] Add support for Audioscrobbler tab-separated file uploads :improvement:
+CLOSED: [2023-02-03 Fri 16:52]
+
+An example of the format:
+#+begin_src csv
+,
+#AUDIOSCROBBLER/1.1
+#TZ/UNKNOWN
+#CLIENT/Rockbox sansaclipplus $Revision$
+75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494944	64ff5f53-d187-4512-827e-7606c69e66ff
+75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494990	64ff5f53-d187-4512-827e-7606c69e66ff
+311	311	Down	1	173	S	1740495003	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Down	1	173	L	1740495049	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Down	1	173	L	1740495113	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Random	2	187	S	1740495190	530c09f3-46fe-4d90-b11f-7b63bcb4b373
+311	311	Random	2	187	L	1740495194	530c09f3-46fe-4d90-b11f-7b63bcb4b373
+311	311	Jackolantern’s Weather	3	204	L	1740495382	cc3b2dec-5d99-47ea-8930-20bf258be4ea
+311	311	All Mixed Up	4	182	L	1740495586	980a78b5-5bdd-4f50-9e3a-e13261e2817b
+311	311	Hive	5	179	L	1740495768	18f6dc98-d3a2-4f81-b967-97359d14c68c
+311	311	Guns (Are for Pussies)	6	137	L	1740495948	5e97ed9f-c8cc-4282-9cbe-f8e17aee5128
+311	311	Misdirected Hostility	7	179	S	1740496085	61ff2c1a-fc9c-44c3-8da1-5e50a44245af
+,
+#+end_src
+* TODO [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
+
+Given a UUID from musicbrainz, we should be able to scrobble an album or
+individual track.
+
+* TODO [#A] Add django-storage to store files on S3 :improvement:
+* TODO [#B] Adjust cancel/finish task to use javascript to submit :improvement:
+* TODO [#B] Implement a detail view for TV shows :improvement:
+* TODO [#B] Implement a detail view for Moviews :improvement:
+* TODO [#B] Allow scrobbling music without MB IDs by grabbing them before scrobble :improvement:
+
+This would allow a few nice flows. One, you'd be able to record the play of an
+entire album by just dropping the muscibrainz_id in. This could be helpful for
+offline listening. It would also mean bad metadata from mopidy would not break
+scrobbling.
+* TODO [#C] Implement keeping track of week/month/year chart-toppers :improvement:
+:LOGBOOK:
+CLOCK: [2023-01-30 Mon 16:30]--[2023-01-30 Mon 18:00] =>  1:30
+:END:
+
+Maloja does this cool thing where artists and tracks get recorded as the top
+track of a given week, month or year. They get gold, silver or bronze stars for
+their place in the time period.
+
+I could see this being implemented as a separate Chart table which gets
+populated at the end of a time period and has a start and end date that defines
+a period, along with a one, two, three instance.
+
+Of course, it could also be a data model without a table, where it runs some fun
+calculations, stores it's values in Redis as a long-term lookup table and just
+has to re-populate when the server restarts.
+* TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :improvement:
+** Example payloads from mopidy-webhooks
+*** Podcast playback ended
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_ended",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 13,
+            "track": {
+                "__model__": "Track",
+                "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
+                "name": "Wolf warriors",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "name": "The Economist"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "name": "The Prince",
+                    "date": "2022"
+                },
+                "genre": "Blues",
+                "date": "2022",
+                "length": 2437778,
+                "bitrate": 127988
+            }
+        },
+        "time_position": 3290
+    }
+}
+#+end_src
+*** Podcast playback state changes
+#+begin_src json
+{
+    "type": "event",
+    "event": "playback_state_changed",
+    "data": {
+        "old_state": "paused",
+        "new_state": "playing"
+    }
+}
+#+end_src
+
+#+begin_src json
+{
+    "type": "event",
+    "event": "playback_state_changed",
+    "data": {
+        "old_state": "stopped",
+        "new_state": "playing"
+    }
+}
+#+end_src
+*** Podcast playback started
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_started",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 13,
+            "track": {
+                "__model__": "Track",
+                "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
+                "name": "Wolf warriors",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "name": "The Economist"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "name": "The Prince",
+                    "date": "2022"
+                },
+                "genre": "Blues",
+                "date": "2022",
+                "length": 2437778,
+                "bitrate": 127988
+            }
+        }
+    }
+}
+#+end_src
+*** Podcast playback paused
+#+begin_src  json
+{
+    "type": "status",
+    "data": {
+        "state": "paused",
+        "current_track": {
+            "__model__": "Track",
+            "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
+            "name": "Wolf warriors",
+            "artists": [
+                {
+                    "__model__": "Artist",
+                    "name": "The Economist"
+                }
+            ],
+            "album": {
+                "__model__": "Album",
+                "name": "The Prince",
+                "date": "2022"
+            },
+            "genre": "Blues",
+            "date": "2022",
+            "length": 2437778,
+            "bitrate": 127988
+        },
+        "time_position": 2350
+    }
+}
+
+#+end_src
+*** Track playback started
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_started",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 14,
+            "track": {
+                "__model__": "Track",
+                "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
+                "name": "Supermassive Black Hole",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
+                        "name": "Muse",
+                        "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
+                    "name": "Twilight: Original Motion Picture Soundtrack",
+                    "artists": [
+                        {
+                            "__model__": "Artist",
+                            "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
+                            "name": "Various Artists",
+                            "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
+                        }
+                    ],
+                    "num_tracks": 12,
+                    "num_discs": 1,
+                    "date": "2008-11-04",
+                    "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
+                },
+                "composers": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
+                        "name": "Matt Bellamy"
+                    }
+                ],
+                "genre": "Rock",
+                "track_no": 1,
+                "disc_no": 1,
+                "date": "2008-11-04",
+                "length": 211121,
+                "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
+                "last_modified": 1672712949510
+            }
+        }
+    }
+}
+#+end_src
+*** Track playback in progress
+#+begin_src json
+{
+    "type": "status",
+    "data": {
+        "state": "playing",
+        "current_track": {
+            "__model__": "Track",
+            "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
+            "name": "Supermassive Black Hole",
+            "artists": [
+                {
+                    "__model__": "Artist",
+                    "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
+                    "name": "Muse",
+                    "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
+                }
+            ],
+            "album": {
+                "__model__": "Album",
+                "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
+                "name": "Twilight: Original Motion Picture Soundtrack",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
+                        "name": "Various Artists",
+                        "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
+                    }
+                ],
+                "num_tracks": 12,
+                "num_discs": 1,
+                "date": "2008-11-04",
+                "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
+            },
+            "composers": [
+                {
+                    "__model__": "Artist",
+                    "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
+                    "name": "Matt Bellamy"
+                }
+            ],
+            "genre": "Rock",
+            "track_no": 1,
+            "disc_no": 1,
+            "date": "2008-11-04",
+            "length": 211121,
+            "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
+            "last_modified": 1672712949510
+        },
+        "time_position": 17031
+    }
+}
+#+end_src
+*** Track event playback paused
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_paused",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 14,
+            "track": {
+                "__model__": "Track",
+                "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
+                "name": "Supermassive Black Hole",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
+                        "name": "Muse",
+                        "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
+                    "name": "Twilight: Original Motion Picture Soundtrack",
+                    "artists": [
+                        {
+                            "__model__": "Artist",
+                            "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
+                            "name": "Various Artists",
+                            "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
+                        }
+                    ],
+                    "num_tracks": 12,
+                    "num_discs": 1,
+                    "date": "2008-11-04",
+                    "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
+                },
+                "composers": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
+                        "name": "Matt Bellamy"
+                    }
+                ],
+                "genre": "Rock",
+                "track_no": 1,
+                "disc_no": 1,
+                "date": "2008-11-04",
+                "length": 211121,
+                "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
+                "last_modified": 1672712949510
+            }
+        },
+        "time_position": 67578
+    }
+}
+#+end_src
+* TODO [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :improvement:
+* TODO Figure out how to add to web-scrobbler :imropvement:
+
+An example:
+https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
+
+* TODO When updating musicbrainz IDs, clear and run fetch artwrok :improvement:

vrobbler.conf.test

@@ -0,0 +1,8 @@
+# Local configuration for Emus
+
+VROBBLER_DUMP_REQUEST_DATA=True
+VROBBLER_LOG_TO_CONSOLE=True
+VROBBLER_DEBUG=True
+VROBBLER_LOG_LEVEL="DEBUG"
+VROBBLER_MEDIA_ROOT = "/tmp/media/"
+VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True

vrobbler/__init__.py

@@ -0,0 +1,5 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

vrobbler/apps/music/admin.py

@@ -2,6 +2,8 @@ from django.contrib import admin
 
 from music.models import Artist, Album, Track
 
+from scrobbles.admin import ScrobbleInline
+
 
 @admin.register(Album)
 class AlbumAdmin(admin.ModelAdmin):
@@ -33,3 +35,6 @@ class TrackAdmin(admin.ModelAdmin):
     )
     list_filter = ("album", "artist")
     ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/music/aggregators.py

@@ -1,12 +1,13 @@
-from django.db.models import Q, Count, Sum
-from typing import List, Optional
-from scrobbles.models import Scrobble
-from music.models import Track, Artist
-from videos.models import Video
-
-from django.utils import timezone
 from datetime import datetime, timedelta
+from typing import List, Optional
 
+import pytz
+from django.db.models import Count, Q, Sum
+from django.utils import timezone
+from music.models import Artist, Track
+from scrobbles.models import Scrobble
+from videos.models import Video
+from vrobbler.apps.profiles.utils import now_user_timezone
 
 NOW = timezone.now()
 START_OF_TODAY = datetime.combine(NOW.date(), datetime.min.time(), NOW.tzinfo)
@@ -17,67 +18,114 @@ STARTING_DAY_OF_CURRENT_MONTH = NOW.date().replace(day=1)
 STARTING_DAY_OF_CURRENT_YEAR = NOW.date().replace(month=1, day=1)
 
 
-def scrobble_counts():
-    finished_scrobbles_qs = Scrobble.objects.filter(played_to_completion=True)
+def scrobble_counts(user=None):
+
+    now = timezone.now()
+    user_filter = Q()
+    if user and user.is_authenticated:
+        now = now_user_timezone(user.profile)
+        user_filter = Q(user=user)
+
+    start_of_today = datetime.combine(
+        now.date(), datetime.min.time(), now.tzinfo
+    )
+    starting_day_of_current_week = now.date() - timedelta(
+        days=now.today().isoweekday() % 7
+    )
+    starting_day_of_current_month = now.date().replace(day=1)
+    starting_day_of_current_year = now.date().replace(month=1, day=1)
+
+    finished_scrobbles_qs = Scrobble.objects.filter(
+        user_filter, played_to_completion=True
+    )
     data = {}
     data['today'] = finished_scrobbles_qs.filter(
-        timestamp__gte=START_OF_TODAY
+        timestamp__gte=start_of_today
     ).count()
     data['week'] = finished_scrobbles_qs.filter(
-        timestamp__gte=STARTING_DAY_OF_CURRENT_WEEK
+        timestamp__gte=starting_day_of_current_week
     ).count()
     data['month'] = finished_scrobbles_qs.filter(
-        timestamp__gte=STARTING_DAY_OF_CURRENT_MONTH
+        timestamp__gte=starting_day_of_current_month
     ).count()
     data['year'] = finished_scrobbles_qs.filter(
-        timestamp__gte=STARTING_DAY_OF_CURRENT_YEAR
+        timestamp__gte=starting_day_of_current_year
     ).count()
     data['alltime'] = finished_scrobbles_qs.count()
     return data
 
 
-def week_of_scrobbles(media: str = 'tracks') -> dict[str, int]:
+def week_of_scrobbles(user=None, media: str = 'tracks') -> dict[str, int]:
+
+    now = timezone.now()
+    user_filter = Q()
+    if user and user.is_authenticated:
+        now = now_user_timezone(user.profile)
+        user_filter = Q(user=user)
+
+    start_of_today = datetime.combine(
+        now.date(), datetime.min.time(), now.tzinfo
+    )
+
     scrobble_day_dict = {}
+    base_qs = Scrobble.objects.filter(user_filter, played_to_completion=True)
+
     media_filter = Q(track__isnull=False)
+    if media == 'movies':
+        media_filter = Q(video__video_type=Video.VideoType.MOVIE)
+    if media == 'series':
+        media_filter = Q(video__video_type=Video.VideoType.TV_EPISODE)
 
     for day in range(6, -1, -1):
-        start = START_OF_TODAY - timedelta(days=day)
-        end = datetime.combine(start, datetime.max.time(), NOW.tzinfo)
+        start = start_of_today - timedelta(days=day)
+        end = datetime.combine(start, datetime.max.time(), now.tzinfo)
         day_of_week = start.strftime('%A')
-        if media == 'movies':
-            media_filter = Q(video__videotype=Video.VideoType.MOVIE)
-        if media == 'series':
-            media_filter = Q(video__videotype=Video.VideoType.TV_EPISODE)
-        scrobble_day_dict[day_of_week] = (
-            Scrobble.objects.filter(media_filter)
-            .filter(
-                timestamp__gte=start,
-                timestamp__lte=end,
-                played_to_completion=True,
-            )
-            .count()
-        )
+
+        scrobble_day_dict[day_of_week] = base_qs.filter(
+            media_filter,
+            timestamp__gte=start,
+            timestamp__lte=end,
+            played_to_completion=True,
+        ).count()
 
     return scrobble_day_dict
 
 
-def top_tracks(filter: str = "today", limit: int = 15) -> List["Track"]:
-    time_filter = Q(scrobble__timestamp__gte=START_OF_TODAY)
+def top_tracks(
+    user: "User", filter: str = "today", limit: int = 15
+) -> List["Track"]:
+
+    now = timezone.now()
+    if user.is_authenticated:
+        now = now_user_timezone(user.profile)
+
+    start_of_today = datetime.combine(
+        now.date(), datetime.min.time(), now.tzinfo
+    )
+    starting_day_of_current_week = now.date() - timedelta(
+        days=now.today().isoweekday() % 7
+    )
+    starting_day_of_current_month = now.date().replace(day=1)
+    starting_day_of_current_year = now.date().replace(month=1, day=1)
+
+    time_filter = Q(scrobble__timestamp__gte=start_of_today)
     if filter == "week":
-        time_filter = Q(scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_WEEK)
+        time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_week)
     if filter == "month":
-        time_filter = Q(scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_MONTH)
+        time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_month)
     if filter == "year":
-        time_filter = Q(scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_YEAR)
+        time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_year)
 
     return (
-        Track.objects.annotate(num_scrobbles=Count("scrobble", distinct=True))
-        .filter(time_filter)
+        Track.objects.filter(time_filter)
+        .annotate(num_scrobbles=Count("scrobble", distinct=True))
         .order_by("-num_scrobbles")[:limit]
     )
 
 
-def top_artists(filter: str = "today", limit: int = 15) -> List["Artist"]:
+def top_artists(
+    user: "User", filter: str = "today", limit: int = 15
+) -> List["Artist"]:
     time_filter = Q(track__scrobble__timestamp__gte=START_OF_TODAY)
     if filter == "week":
         time_filter = Q(
@@ -93,10 +141,8 @@ def top_artists(filter: str = "today", limit: int = 15) -> List["Artist"]:
         )
 
     return (
-        Artist.objects.annotate(
-            num_scrobbles=Count("track__scrobble", distinct=True)
-        )
-        .filter(time_filter)
+        Artist.objects.filter(time_filter)
+        .annotate(num_scrobbles=Count("track__scrobble", distinct=True))
         .order_by("-num_scrobbles")[:limit]
     )
 

vrobbler/apps/music/migrations/0009_alter_track_musicbrainz_id_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.5 on 2023-01-19 20:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0008_alter_track_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='track',
+            name='musicbrainz_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='track',
+            unique_together={('album', 'musicbrainz_id')},
+        ),
+    ]

vrobbler/apps/music/models.py

@@ -3,9 +3,10 @@ from typing import Dict, Optional
 from uuid import uuid4
 
 import musicbrainzngs
-from django.apps.config import cached_property
+from django.conf import settings
 from django.core.files.base import ContentFile
 from django.db import models
+from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
 from scrobbles.mixins import ScrobblableMixin
@@ -29,6 +30,29 @@ class Artist(TimeStampedModel):
     def mb_link(self):
         return f"https://musicbrainz.org/artist/{self.musicbrainz_id}"
 
+    def get_absolute_url(self):
+        return reverse('music:artist_detail', kwargs={'slug': self.uuid})
+
+    def scrobbles(self):
+        from scrobbles.models import Scrobble
+
+        return Scrobble.objects.filter(
+            track__in=self.track_set.all()
+        ).order_by('-timestamp')
+
+    @property
+    def tracks(self):
+        return (
+            self.track_set.all()
+            .annotate(scrobble_count=models.Count('scrobble'))
+            .order_by('-scrobble_count')
+        )
+
+    def charts(self):
+        from scrobbles.models import ChartRecord
+
+        return ChartRecord.objects.filter(track__artist=self).order_by('-year')
+
 
 class Album(TimeStampedModel):
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
@@ -48,18 +72,38 @@ class Album(TimeStampedModel):
         return self.artists.first()
 
     def fix_metadata(self):
-        if not self.musicbrainz_albumartist_id or not self.year:
+        if (
+            not self.musicbrainz_albumartist_id
+            or not self.year
+            or not self.musicbrainz_releasegroup_id
+        ):
             musicbrainzngs.set_useragent('vrobbler', '0.3.0')
             mb_data = musicbrainzngs.get_release_by_id(
-                self.musicbrainz_id, includes=['artists']
+                self.musicbrainz_id, includes=['artists', 'release-groups']
             )
+            if not self.musicbrainz_releasegroup_id:
+                self.musicbrainz_releasegroup_id = mb_data['release'][
+                    'release-group'
+                ]['id']
             if not self.musicbrainz_albumartist_id:
                 self.musicbrainz_albumartist_id = mb_data['release'][
                     'artist-credit'
                 ][0]['artist']['id']
             if not self.year:
-                self.year = mb_data['release']['date'][0:4]
-            self.save(update_fields=['musicbrainz_albumartist_id', 'year'])
+                try:
+                    self.year = mb_data['release']['date'][0:4]
+                except KeyError:
+                    pass
+                except IndexError:
+                    pass
+
+            self.save(
+                update_fields=[
+                    'musicbrainz_albumartist_id',
+                    'musicbrainz_releasegroup_id',
+                    'year',
+                ]
+            )
 
             new_artist = Artist.objects.filter(
                 musicbrainz_id=self.musicbrainz_albumartist_id
@@ -69,16 +113,46 @@ class Album(TimeStampedModel):
             if not new_artist:
                 for t in self.track_set.all():
                     self.artists.add(t.artist)
+            if (
+                not self.cover_image
+                or self.cover_image == 'default-image-replace-me'
+            ):
+                self.fetch_artwork()
 
-    def fetch_artwork(self):
-        if not self.cover_image:
-            try:
-                img_data = musicbrainzngs.get_image_front(self.musicbrainz_id)
-                name = f"{self.name}_{self.uuid}.jpg"
-                self.cover_image = ContentFile(img_data, name=name)
-            except musicbrainzngs.ResponseError:
-                logger.warning(f'No cover art found for {self.name}')
-                self.cover_image = 'default-image-replace-me'
+    def fetch_artwork(self, force=False):
+        if not self.cover_image and not force:
+            if self.musicbrainz_id:
+                try:
+                    img_data = musicbrainzngs.get_image_front(
+                        self.musicbrainz_id
+                    )
+                    name = f"{self.name}_{self.uuid}.jpg"
+                    self.cover_image = ContentFile(img_data, name=name)
+                    logger.info(f'Setting image to {name}')
+                except musicbrainzngs.ResponseError:
+                    logger.warning(
+                        f'No cover art found for {self.name} by release'
+                    )
+
+            if (
+                not self.cover_image
+                or self.cover_image == "default-image-replace-me"
+            ) and self.musicbrainz_releasegroup_id:
+                try:
+                    img_data = musicbrainzngs.get_release_group_image_front(
+                        self.musicbrainz_releasegroup_id
+                    )
+                    name = f"{self.name}_{self.uuid}.jpg"
+                    self.cover_image = ContentFile(img_data, name=name)
+                    logger.info(f'Setting image to {name}')
+                except musicbrainzngs.ResponseError:
+                    logger.warning(
+                        f'No cover art found for {self.name} by release group'
+                    )
+            if not self.cover_image:
+                logger.debug(
+                    f"No cover art found for release or release group for {self.name}, setting to default"
+                )
             self.save()
 
     @property
@@ -87,6 +161,8 @@ class Album(TimeStampedModel):
 
 
 class Track(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, 'MUSIC_COMPLETION_PERCENT', 90)
+
     class Opinion(models.IntegerChoices):
         DOWN = -1, 'Thumbs down'
         NEUTRAL = 0, 'No opinion'
@@ -94,19 +170,21 @@ class Track(ScrobblableMixin):
 
     artist = models.ForeignKey(Artist, on_delete=models.DO_NOTHING)
     album = models.ForeignKey(Album, on_delete=models.DO_NOTHING, **BNULL)
-    musicbrainz_id = models.CharField(max_length=255, unique=True, **BNULL)
+    musicbrainz_id = models.CharField(max_length=255, **BNULL)
+
+    class Meta:
+        unique_together = [['album', 'musicbrainz_id']]
 
     def __str__(self):
         return f"{self.title} by {self.artist}"
 
+    def get_absolute_url(self):
+        return reverse('music:track_detail', kwargs={'slug': self.uuid})
+
     @property
     def mb_link(self):
         return f"https://musicbrainz.org/recording/{self.musicbrainz_id}"
 
-    @cached_property
-    def scrobble_count(self):
-        return self.scrobble_set.filter(in_progress=False).count()
-
     @classmethod
     def find_or_create(
         cls, artist_dict: Dict, album_dict: Dict, track_dict: Dict
@@ -125,16 +203,7 @@ class Track(ScrobblableMixin):
             return
 
         artist, artist_created = Artist.objects.get_or_create(**artist_dict)
-        if artist_created:
-            logger.debug(f"Created new album {artist}")
-        else:
-            logger.debug(f"Found album {artist}")
-
         album, album_created = Album.objects.get_or_create(**album_dict)
-        if album_created:
-            logger.debug(f"Created new album {album}")
-        else:
-            logger.debug(f"Found album {album}")
 
         album.fix_metadata()
         if not album.cover_image:
@@ -144,9 +213,5 @@ class Track(ScrobblableMixin):
         track_dict['artist_id'] = artist.id
 
         track, created = cls.objects.get_or_create(**track_dict)
-        if created:
-            logger.debug(f"Created new track: {track}")
-        else:
-            logger.debug(f"Found track {track}")
 
         return track

vrobbler/apps/music/urls.py

@@ -0,0 +1,21 @@
+from django.urls import path
+from music import views
+
+app_name = 'music'
+
+
+urlpatterns = [
+    path('albums/', views.AlbumListView.as_view(), name='albums_list'),
+    path("tracks/", views.TrackListView.as_view(), name='tracks_list'),
+    path(
+        'tracks/<slug:slug>/',
+        views.TrackDetailView.as_view(),
+        name='track_detail',
+    ),
+    path('artists/', views.ArtistListView.as_view(), name='artist_list'),
+    path(
+        'artists/<slug:slug>/',
+        views.ArtistDetailView.as_view(),
+        name='artist_detail',
+    ),
+]

vrobbler/apps/music/utils.py

@@ -0,0 +1,106 @@
+import re
+import logging
+
+from scrobbles.musicbrainz import (
+    lookup_album_dict_from_mb,
+    lookup_artist_from_mb,
+)
+
+logger = logging.getLogger(__name__)
+
+
+from music.models import Artist, Album, Track
+
+
+def get_or_create_artist(name: str, mbid: str = None) -> Artist:
+    artist = None
+    logger.debug(f'Got artist {name} and mbid: {mbid}')
+
+    if 'feat.' in name.lower():
+        name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip()
+    if 'featuring' in name.lower():
+        name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip()
+    artist_dict = lookup_artist_from_mb(name)
+    mbid = mbid or artist_dict['id']
+
+    logger.debug(f'Looking up artist {name} and mbid: {mbid}')
+    artist, created = Artist.objects.get_or_create(
+        name=name, musicbrainz_id=mbid
+    )
+
+    logger.debug(f"Cleaning artist {name} with {artist_dict['name']}")
+    # Clean up bad names in our DB with MB names
+    if artist.name != artist_dict['name']:
+        artist.name = artist_dict["name"]
+        artist.save(update_fields=["name"])
+
+    return artist
+
+
+def get_or_create_album(name: str, artist: Artist, mbid: str = None) -> Album:
+    album = None
+    album_created = False
+    albums = Album.objects.filter(name__iexact=name)
+    if albums.count() == 1:
+        album = albums.first()
+    else:
+        for potential_album in albums:
+            if artist in album.artist_set.all():
+                album = potential_album
+    if not album:
+        album_created = True
+        album = Album.objects.create(name=name, musicbrainz_id=mbid)
+        album.save()
+        album.artists.add(artist)
+
+    if album_created or not mbid:
+        album_dict = lookup_album_dict_from_mb(
+            album.name, artist_name=artist.name
+        )
+        album.year = album_dict["year"]
+        album.musicbrainz_id = album_dict["mb_id"]
+        album.musicbrainz_releasegroup_id = album_dict["mb_group_id"]
+        album.musicbrainz_albumartist_id = artist.musicbrainz_id
+        album.save(
+            update_fields=[
+                "year",
+                "musicbrainz_id",
+                "musicbrainz_releasegroup_id",
+                "musicbrainz_albumartist_id",
+            ]
+        )
+        album.artists.add(artist)
+        album.fetch_artwork()
+    return album
+
+
+def get_or_create_track(
+    title: str,
+    mbid: str,
+    artist: Artist,
+    album: Album,
+    run_time=None,
+    run_time_ticks=None,
+) -> Track:
+    track = None
+    if mbid:
+        track = Track.objects.filter(
+            musicbrainz_id=mbid,
+        ).first()
+    if not track:
+        track = Track.objects.filter(
+            title=title, artist=artist, album=album
+        ).first()
+
+    # TODO Can we look up mbid for tracks?
+    if not track:
+        track = Track.objects.create(
+            title=title,
+            artist=artist,
+            album=album,
+            musicbrainz_id=mbid,
+            run_time=run_time,
+            run_time_ticks=run_time_ticks,
+        )
+
+    return track

vrobbler/apps/music/views.py

@@ -0,0 +1,33 @@
+from django.views import generic
+from music.models import Track, Artist, Album
+from scrobbles.stats import get_scrobble_count_qs
+
+
+class TrackListView(generic.ListView):
+    model = Track
+
+    def get_queryset(self):
+        return get_scrobble_count_qs(user=self.request.user).order_by(
+            "-scrobble_count"
+        )
+
+
+class TrackDetailView(generic.DetailView):
+    model = Track
+    slug_field = 'uuid'
+
+
+class ArtistListView(generic.ListView):
+    model = Artist
+
+    def get_queryset(self):
+        return super().get_queryset().order_by("name")
+
+
+class ArtistDetailView(generic.DetailView):
+    model = Artist
+    slug_field = 'uuid'
+
+
+class AlbumListView(generic.ListView):
+    model = Album

vrobbler/apps/podcasts/admin.py

@@ -1,6 +1,6 @@
-#!/usr/bin/env python3
 from django.contrib import admin
 from podcasts.models import Episode, Podcast, Producer
+from scrobbles.admin import ScrobbleInline
 
 
 @admin.register(Producer)
@@ -31,3 +31,6 @@ class EpisodeAdmin(admin.ModelAdmin):
     )
     list_filter = ("podcast",)
     ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/podcasts/models.py

@@ -2,11 +2,12 @@ import logging
 from typing import Dict, Optional
 from uuid import uuid4
 
+from django.conf import settings
 from django.db import models
 from django.utils.translation import gettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
 
-from vrobbler.apps.scrobbles.mixins import ScrobblableMixin
+from scrobbles.mixins import ScrobblableMixin
 
 logger = logging.getLogger(__name__)
 BNULL = {"blank": True, "null": True}
@@ -34,6 +35,8 @@ class Podcast(TimeStampedModel):
 
 
 class Episode(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, 'PODCAST_COMPLETION_PERCENT', 90)
+
     podcast = models.ForeignKey(Podcast, on_delete=models.DO_NOTHING)
     number = models.IntegerField(**BNULL)
     pub_date = models.DateField(**BNULL)

vrobbler/apps/profiles/admin.py

@@ -0,0 +1,9 @@
+from django.contrib import admin
+
+from profiles.models import UserProfile
+
+
+@admin.register(UserProfile)
+class UserProfileAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    ordering = ("-created",)

vrobbler/apps/profiles/constants.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+
+import pytz
+
+ALL_TIMEZONE_CHOICES = tuple(zip(pytz.all_timezones, pytz.all_timezones))
+COMMON_TIMEZONE_CHOICES = tuple(
+    zip(pytz.common_timezones, pytz.common_timezones)
+)
+PRETTY_TIMEZONE_CHOICES = []
+
+for tz in pytz.common_timezones:
+    now = datetime.now(pytz.timezone(tz))
+    ofs = now.strftime("%z")
+    PRETTY_TIMEZONE_CHOICES.append((int(ofs), tz, "(GMT%s) %s" % (ofs, tz)))
+PRETTY_TIMEZONE_CHOICES.sort()
+for i in range(len(PRETTY_TIMEZONE_CHOICES)):
+    PRETTY_TIMEZONE_CHOICES[i] = PRETTY_TIMEZONE_CHOICES[i][1:]

vrobbler/apps/profiles/migrations/0002_userprofile_lastfm_password_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.5 on 2023-02-12 22:26
+
+from django.db import migrations, models
+import encrypted_field.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('profiles', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='userprofile',
+            name='lastfm_password',
+            field=encrypted_field.fields.EncryptedField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='userprofile',
+            name='lastfm_username',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/profiles/models.py

@@ -0,0 +1,29 @@
+import pytz
+
+from django.contrib.auth import get_user_model
+from django.db import models
+from django_extensions.db.models import TimeStampedModel
+from profiles.constants import PRETTY_TIMEZONE_CHOICES
+
+from encrypted_field import EncryptedField
+
+User = get_user_model()
+BNULL = {"blank": True, "null": True}
+
+
+class UserProfile(TimeStampedModel):
+    user = models.OneToOneField(
+        User, on_delete=models.CASCADE, related_name="profile"
+    )
+    timezone = models.CharField(
+        max_length=255, choices=PRETTY_TIMEZONE_CHOICES
+    )
+    lastfm_username = models.CharField(max_length=255, **BNULL)
+    lastfm_password = EncryptedField(**BNULL)
+
+    def __str__(self):
+        return f"User profile for {self.user}"
+
+    @property
+    def tzinfo(self):
+        return pytz.timezone(self.timezone)

vrobbler/apps/profiles/signals.py

@@ -0,0 +1,13 @@
+from django.contrib.auth import get_user_model
+from django.db.models.base import post_save
+from django.dispatch import receiver
+
+from profiles.models import UserProfile
+
+User = get_user_model()
+
+
+@receiver(post_save, sender=User)
+def create_profile(sender, instance, created, **kwargs):
+    if created:
+        UserProfile.objects.create(user=instance)

vrobbler/apps/profiles/utils.py

@@ -0,0 +1,33 @@
+import datetime
+
+import pytz
+from django.conf import settings
+from django.utils import timezone
+
+
+# need to translate to a non-naive timezone, even if timezone == settings.TIME_ZONE, so we can compare two dates
+def to_user_timezone(date, profile):
+    timezone = profile.timezone if profile.timezone else settings.TIME_ZONE
+    return date.replace(tzinfo=pytz.timezone(settings.TIME_ZONE)).astimezone(
+        pytz.timezone(timezone)
+    )
+
+
+def to_system_timezone(date, profile):
+    timezone = profile.timezone if profile.timezone else settings.TIME_ZONE
+    return date.replace(tzinfo=pytz.timezone(timezone)).astimezone(
+        pytz.timezone(settings.TIME_ZONE)
+    )
+
+
+def now_user_timezone(profile):
+    timezone.activate(pytz.timezone(profile.timezone))
+    return timezone.localtime(timezone.now())
+
+
+def now_system_timezone():
+    return (
+        datetime.datetime.now()
+        .replace(tzinfo=pytz.timezone(settings.TIME_ZONE))
+        .astimezone(pytz.timezone(settings.TIME_ZONE))
+    )

vrobbler/apps/scrobbles/admin.py

@@ -1,6 +1,61 @@
 from django.contrib import admin
+from scrobbles.models import (
+    AudioScrobblerTSVImport,
+    ChartRecord,
+    LastFmImport,
+    Scrobble,
+)
 
-from scrobbles.models import Scrobble
+
+class ScrobbleInline(admin.TabularInline):
+    model = Scrobble
+    extra = 0
+    raw_id_fields = ('video', 'podcast_episode', 'track')
+    exclude = ('source_id', 'scrobble_log')
+
+
+@admin.register(AudioScrobblerTSVImport)
+class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("uuid", "created", "process_count", "tsv_file")
+    ordering = ("-created",)
+
+
+@admin.register(LastFmImport)
+class LastFmImportAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "uuid",
+        "process_count",
+        "processed_finished",
+        "processing_started",
+    )
+    ordering = ("-created",)
+
+
+@admin.register(ChartRecord)
+class ChartRecordAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "user",
+        "rank",
+        "year",
+        "week",
+        "month",
+        "day",
+        "media_name",
+    )
+    ordering = ("-created",)
+
+    def media_name(self, obj):
+        if obj.video:
+            return obj.video
+        if obj.track:
+            return obj.track
+        if obj.podcast_episode:
+            return obj.podcast_episode
+        if obj.sport_event:
+            return obj.sport_event
 
 
 @admin.register(Scrobble)
@@ -16,6 +71,7 @@ class ScrobbleAdmin(admin.ModelAdmin):
         "is_paused",
         "played_to_completion",
     )
+    raw_id_fields = ('video', 'podcast_episode', 'track', 'sport_event')
     list_filter = ("is_paused", "in_progress", "source", "track__artist")
     ordering = ("-timestamp",)
 
@@ -26,6 +82,8 @@ class ScrobbleAdmin(admin.ModelAdmin):
             return obj.track
         if obj.podcast_episode:
             return obj.podcast_episode
+        if obj.sport_event:
+            return obj.sport_event
 
     def media_type(self, obj):
         if obj.video:
@@ -34,6 +92,8 @@ class ScrobbleAdmin(admin.ModelAdmin):
             return "Track"
         if obj.podcast_episode:
             return "Podcast"
+        if obj.sport_event:
+            return "Sport Event"
 
     def playback_percent(self, obj):
         return obj.percent_played

vrobbler/apps/scrobbles/constants.py

@@ -1,3 +1,2 @@
-#!/usr/bin/env python3
 JELLYFIN_VIDEO_ITEM_TYPES = ["Episode", "Movie"]
 JELLYFIN_AUDIO_ITEM_TYPES = ["Audio"]

vrobbler/apps/scrobbles/export.py

@@ -0,0 +1,67 @@
+import csv
+import tempfile
+from scrobbles.models import Scrobble
+
+from django.db.models import Q
+
+
+def export_scrobbles(start_date=None, end_date=None, format="AS"):
+    start_query = Q()
+    end_query = Q()
+    if start_date:
+        start_query = Q(timestamp__gte=start_date)
+    if start_date:
+        end_query = Q(timestamp__lte=end_date)
+
+    scrobble_qs = Scrobble.objects.filter(
+        start_query, end_query, track__isnull=False
+    )
+    headers = []
+    extension = 'tsv'
+    delimiter = '\t'
+
+    if format == "as":
+        headers = [
+            ['#AUDIOSCROBBLER/1.1'],
+            ['#TZ/UTC'],
+            ['#CLIENT/Vrobbler 1.0.0'],
+        ]
+
+    if format == "csv":
+        delimiter = ','
+        extension = 'csv'
+        headers = [
+            [
+                "artists",
+                "album",
+                "title",
+                "track_number",
+                "run_time",
+                "rating",
+                "timestamp",
+                "musicbrainz_id",
+            ]
+        ]
+
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as outfile:
+        writer = csv.writer(outfile, delimiter=delimiter)
+        for row in headers:
+            writer.writerow(row)
+
+        for scrobble in scrobble_qs:
+            track = scrobble.track
+            track_number = 0  # TODO Add track number
+            track_rating = "S"  # TODO implement ratings?
+            track_artist = track.artist or track.album.primary_artist
+            row = [
+                track_artist,
+                track.album.name,
+                track.title,
+                track_number,
+                track.run_time,
+                track_rating,
+                scrobble.timestamp.strftime('%s'),
+                track.musicbrainz_id,
+            ]
+            writer.writerow(row)
+        return outfile.name, extension

vrobbler/apps/scrobbles/forms.py

@@ -0,0 +1,25 @@
+from django import forms
+
+
+class ExportScrobbleForm(forms.Form):
+    """Provide options for downloading scrobbles"""
+
+    EXPORT_TYPES = (
+        ('as', 'Audioscrobbler'),
+        ('csv', 'CSV'),
+        ('html', 'HTML'),
+    )
+    export_type = forms.ChoiceField(choices=EXPORT_TYPES)
+
+
+class ScrobbleForm(forms.Form):
+    item_id = forms.CharField(
+        label="",
+        widget=forms.TextInput(
+            attrs={
+                'class': "form-control form-control-dark w-100",
+                'placeholder': "Scrobble something (IMDB ID, String, TVDB ID ...)",
+                'aria-label': "Scrobble something",
+            }
+        ),
+    )

vrobbler/apps/scrobbles/imdb.py

@@ -0,0 +1,61 @@
+import logging
+from django.utils import timezone
+
+from imdb import Cinemagoer
+
+imdb_client = Cinemagoer()
+
+logger = logging.getLogger(__name__)
+
+
+def lookup_video_from_imdb(imdb_id: str) -> dict:
+
+    if 'tt' not in imdb_id:
+        logger.warning(f"IMDB ID should begin with 'tt' {imdb_id}")
+        return
+
+    lookup_id = imdb_id.strip('tt')
+    media = imdb_client.get_movie(lookup_id)
+
+    run_time_seconds = 60 * 60
+    runtimes = media.get("runtimes")
+    if runtimes:
+        run_time_seconds = int(runtimes[0]) * 60
+
+    # Ticks otherwise known as miliseconds
+    run_time_ticks = run_time_seconds * 1000 * 1000
+
+    item_type = "Movie"
+    if media.get('series title'):
+        item_type = "Episode"
+
+    try:
+        plot = media.get('plot')[0]
+    except TypeError:
+        plot = ""
+    except IndexError:
+        plot = ""
+
+    logger.debug(f"Received data from IMDB: {media.__dict__}")
+    # Build a rough approximation of a Jellyfin data response
+    data_dict = {
+        "ItemType": item_type,
+        "Name": media.get('title'),
+        "Overview": plot,
+        "Tagline": media.get('tagline'),
+        "Year": media.get('year'),
+        "Provider_imdb": imdb_id,
+        "RunTime": run_time_seconds,
+        "RunTimeTicks": run_time_ticks,
+        "SeriesName": media.get('series title'),
+        "EpisodeNumber": media.get('episode'),
+        "SeasonNumber": media.get('season'),
+        "PlaybackPositionTicks": 1,
+        "PlaybackPosition": 1,
+        "UtcTimestamp": timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f%z'),
+        "IsPaused": False,
+        "PlayedToCompletion": False,
+    }
+    logger.debug(f"Parsed data from IMDB data: {data_dict}")
+
+    return data_dict

vrobbler/apps/scrobbles/lastfm.py

@@ -0,0 +1,167 @@
+import logging
+import time
+from datetime import datetime, timedelta
+
+import pylast
+import pytz
+from django.conf import settings
+from django.utils import timezone
+from music.utils import (
+    get_or_create_album,
+    get_or_create_artist,
+    get_or_create_track,
+)
+
+logger = logging.getLogger(__name__)
+
+PYLAST_ERRORS = tuple(
+    getattr(pylast, exc_name)
+    for exc_name in (
+        "ScrobblingError",
+        "NetworkError",
+        "MalformedResponseError",
+        "WSError",
+    )
+    if hasattr(pylast, exc_name)
+)
+
+
+class LastFM:
+    def __init__(self, user):
+        try:
+            self.client = pylast.LastFMNetwork(
+                api_key=getattr(settings, "LASTFM_API_KEY"),
+                api_secret=getattr(settings, "LASTFM_SECRET_KEY"),
+                username=user.profile.lastfm_username,
+                password_hash=pylast.md5(user.profile.lastfm_password),
+            )
+            self.user = self.client.get_user(user.profile.lastfm_username)
+            self.vrobbler_user = user
+        except PYLAST_ERRORS as e:
+            logger.error(f"Error during Last.fm setup: {e}")
+
+    def import_from_lastfm(self, last_processed=None):
+        """Given a last processed time, import all scrobbles from LastFM since then"""
+        from scrobbles.models import Scrobble
+
+        new_scrobbles = []
+        source = "Last.fm"
+        source_id = ""
+        latest_scrobbles = self.get_last_scrobbles(time_from=last_processed)
+
+        for scrobble in latest_scrobbles:
+            timestamp = scrobble.pop('timestamp')
+
+            artist = get_or_create_artist(scrobble.pop('artist'))
+            album = get_or_create_album(scrobble.pop('album'), artist)
+
+            scrobble['artist'] = artist
+            scrobble['album'] = album
+            track = get_or_create_track(**scrobble)
+
+            new_scrobble = Scrobble(
+                user=self.vrobbler_user,
+                timestamp=timestamp,
+                source=source,
+                source_id=source_id,
+                track=track,
+                played_to_completion=True,
+                in_progress=False,
+            )
+            # Vrobbler scrobbles on finish, LastFM scrobbles on start
+            seconds_eariler = timestamp - timedelta(seconds=20)
+            seconds_later = timestamp + timedelta(seconds=20)
+            existing = Scrobble.objects.filter(
+                created__gte=seconds_eariler,
+                created__lte=seconds_later,
+                track=track,
+            ).first()
+            if existing:
+                logger.debug(f"Skipping existing scrobble {new_scrobble}")
+                continue
+            logger.debug(f"Queued scrobble {new_scrobble} for creation")
+            new_scrobbles.append(new_scrobble)
+
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={'created_scrobbles': created},
+        )
+        return created
+
+    @staticmethod
+    def undo_lastfm_import(process_log, dryrun=True):
+        """Given a newline separated list of scrobbles, delete them"""
+        from scrobbles.models import Scrobble
+
+        if not process_log:
+            logger.warning("No lines in process log found to undo")
+            return
+
+        for line in process_log.split('\n'):
+            scrobble_id = line.split("\t")[0]
+            scrobble = Scrobble.objects.filter(id=scrobble_id).first()
+            if not scrobble:
+                logger.warning(
+                    f"Could not find scrobble {scrobble_id} to undo"
+                )
+                continue
+            logger.info(f"Removing scrobble {scrobble_id}")
+            if not dryrun:
+                scrobble.delete()
+
+    def get_last_scrobbles(self, time_from=None, time_to=None):
+        """Given a user, Last.fm api key, and secret key, grab a list of scrobbled
+        tracks"""
+        lfm_params = {}
+        scrobbles = []
+        if time_from:
+            lfm_params["time_from"] = int(time_from.timestamp())
+        if time_to:
+            lfm_params["time_to"] = int(time_to.timestamp())
+
+        # if not time_from and not time_to:
+        lfm_params['limit'] = None
+
+        found_scrobbles = self.user.get_recent_tracks(**lfm_params)
+        # TOOD spin this out into a celery task over certain threshold of found scrobbles?
+
+        for scrobble in found_scrobbles:
+            run_time = None
+            run_time_ticks = None
+            mbid = None
+            artist = None
+
+            try:
+                run_time_ticks = scrobble.track.get_duration()
+                run_time = int(run_time_ticks / 1000)
+                mbid = scrobble.track.get_mbid()
+                artist = scrobble.track.get_artist().name
+            except pylast.MalformedResponseError as e:
+                logger.warn(e)
+            except pylast.WSError as e:
+                logger.warn(
+                    "LastFM barfed trying to get the track for {scrobble.track}"
+                )
+
+            if not mbid or not artist:
+                logger.warn(f"Silly LastFM, bad data, bailing on {scrobble}")
+                continue
+
+            timestamp = datetime.utcfromtimestamp(
+                int(scrobble.timestamp)
+            ).replace(tzinfo=pytz.utc)
+
+            logger.info(f"{artist},{scrobble.track.title},{timestamp}")
+            scrobbles.append(
+                {
+                    "artist": artist,
+                    "album": scrobble.album,
+                    "title": scrobble.track.title,
+                    "mbid": mbid,
+                    "run_time": run_time,
+                    "run_time_ticks": run_time_ticks,
+                    "timestamp": timestamp,
+                }
+            )
+        return scrobbles

vrobbler/apps/scrobbles/migrations/0008_scrobble_sport_event.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-01-14 21:23
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sports', '0002_rename_start_utc_sportevent_start'),
+        ('scrobbles', '0007_scrobble_podcast_episode'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scrobble',
+            name='sport_event',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to='sports.sportevent',
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0009_scrobble_uuid.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.5 on 2023-01-20 18:40
+
+from uuid import uuid4
+from django.db import migrations, models
+
+
+def generate_uuids(apps, schema_editor):
+    """Force uuid generation for old scrobbles"""
+    Scrobble = apps.get_model('scrobbles', 'Scrobble')
+    for scrobble in Scrobble.objects.all():
+        if not scrobble.uuid:
+            scrobble.uuid = uuid4()
+            scrobble.save(update_fields=['uuid'])
+
+
+def reverse_generate_uuids(apps, schema_editor):
+    pass
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0008_scrobble_sport_event'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scrobble',
+            name='uuid',
+            field=models.UUIDField(blank=True, editable=False, null=True),
+        ),
+        migrations.RunPython(
+            code=generate_uuids, reverse_code=reverse_generate_uuids
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0010_chartrecord.py

@@ -0,0 +1,88 @@
+# Generated by Django 4.1.5 on 2023-01-30 17:01
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0009_alter_track_musicbrainz_id_and_more'),
+        ('videos', '0006_alter_video_year'),
+        ('scrobbles', '0009_scrobble_uuid'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ChartRecord',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('rank', models.IntegerField()),
+                ('year', models.IntegerField(default=2023)),
+                ('month', models.IntegerField(blank=True, null=True)),
+                ('week', models.IntegerField(blank=True, null=True)),
+                ('day', models.IntegerField(blank=True, null=True)),
+                (
+                    'artist',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='music.artist',
+                    ),
+                ),
+                (
+                    'series',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='videos.series',
+                    ),
+                ),
+                (
+                    'track',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='music.track',
+                    ),
+                ),
+                (
+                    'video',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='videos.video',
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0011_chartrecord_user.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.5 on 2023-01-30 17:44
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('scrobbles', '0010_chartrecord'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='chartrecord',
+            name='user',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0012_audioscrobblertsvimport.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.1.5 on 2023-02-03 19:50
+
+from django.db import migrations, models
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0011_chartrecord_user'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AudioScrobblerTSVImport',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'tsv_file',
+                    models.FileField(
+                        blank=True,
+                        null=True,
+                        upload_to='audioscrobbler-uploads/%Y/%m-%d/',
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0013_audioscrobblertsvimport_processed_on.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-03 20:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0012_audioscrobblertsvimport'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='processed_on',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0014_audioscrobblertsvimport_process_log.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-03 22:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0013_audioscrobblertsvimport_processed_on'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='process_log',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0015_audioscrobblertsvimport_uuid_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.1.5 on 2023-02-03 23:36
+
+from django.db import migrations, models
+import scrobbles.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0014_audioscrobblertsvimport_process_log'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='audioscrobblertsvimport',
+            name='tsv_file',
+            field=models.FileField(
+                blank=True,
+                null=True,
+                upload_to=scrobbles.models.AudioScrobblerTSVImport.get_path,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0016_audioscrobblertsvimport_process_count.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-03 23:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0015_audioscrobblertsvimport_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='process_count',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0017_audioscrobblertsvimport_user.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.5 on 2023-02-07 00:07
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('scrobbles', '0016_audioscrobblertsvimport_process_count'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='user',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to=settings.AUTH_USER_MODEL,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0018_lastfmimport.py

@@ -0,0 +1,61 @@
+# Generated by Django 4.1.5 on 2023-02-13 06:43
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('scrobbles', '0017_audioscrobblertsvimport_user'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LastFmImport',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)),
+                ('processed_on', models.DateTimeField(blank=True, null=True)),
+                ('process_log', models.TextField(blank=True, null=True)),
+                ('process_count', models.IntegerField(blank=True, null=True)),
+                (
+                    'user',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0019_rename_processed_on_lastfmimport_processed_finished_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-02-16 17:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0018_lastfmimport'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='lastfmimport',
+            old_name='processed_on',
+            new_name='processed_finished',
+        ),
+        migrations.AddField(
+            model_name='lastfmimport',
+            name='processing_started',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/mixins.py

@@ -7,6 +7,8 @@ BNULL = {"blank": True, "null": True}
 
 
 class ScrobblableMixin(TimeStampedModel):
+    SECONDS_TO_STALE = 1600
+
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     title = models.CharField(max_length=255, **BNULL)
     run_time = models.CharField(max_length=8, **BNULL)

vrobbler/apps/scrobbles/models.py

@@ -1,32 +1,236 @@
+import calendar
 import logging
-from datetime import timedelta
-from typing import Optional
+from uuid import uuid4
 
-from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import models
 from django.utils import timezone
 from django_extensions.db.models import TimeStampedModel
-from music.models import Track
+from music.models import Artist, Track
 from podcasts.models import Episode
+from profiles.utils import now_user_timezone
+from scrobbles.lastfm import LastFM
 from scrobbles.utils import check_scrobble_for_finish
-from videos.models import Video
+from sports.models import SportEvent
+from videos.models import Series, Video
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
 BNULL = {"blank": True, "null": True}
-VIDEO_BACKOFF = getattr(settings, 'VIDEO_BACKOFF_MINUTES')
-TRACK_BACKOFF = getattr(settings, 'MUSIC_BACKOFF_SECONDS')
-VIDEO_WAIT_PERIOD = getattr(settings, 'VIDEO_WAIT_PERIOD_DAYS')
-TRACK_WAIT_PERIOD = getattr(settings, 'MUSIC_WAIT_PERIOD_MINUTES')
+
+
+class AudioScrobblerTSVImport(TimeStampedModel):
+    def get_path(instance, filename):
+        extension = filename.split('.')[-1]
+        uuid = instance.uuid
+        return f'audioscrobbler-uploads/{uuid}.{extension}'
+
+    user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
+    uuid = models.UUIDField(editable=False, default=uuid4)
+    tsv_file = models.FileField(upload_to=get_path, **BNULL)
+    processed_on = models.DateTimeField(**BNULL)
+    process_log = models.TextField(**BNULL)
+    process_count = models.IntegerField(**BNULL)
+
+    def __str__(self):
+        if self.tsv_file:
+            return f"Audioscrobbler TSV upload: {self.tsv_file.path}"
+        return f"Audioscrobbler TSV upload {self.id}"
+
+    def process(self, force=False):
+        from scrobbles.tsv import process_audioscrobbler_tsv_file
+
+        if self.processed_on and not force:
+            logger.info(f"{self} already processed on {self.processed_on}")
+            return
+
+        tz = None
+        if self.user:
+            tz = self.user.profile.tzinfo
+        scrobbles = process_audioscrobbler_tsv_file(
+            self.tsv_file.path, user_tz=tz
+        )
+        self.process_log = ""
+        if scrobbles:
+            for count, scrobble in enumerate(scrobbles):
+                scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}"
+                log_line = f"{scrobble_str}"
+                if count > 0:
+                    log_line = "\n" + log_line
+                self.process_log += log_line
+            self.process_count = len(scrobbles)
+        else:
+            self.process_log = f"Created no new scrobbles"
+            self.process_count = 0
+
+        self.processed_on = timezone.now()
+        self.save(
+            update_fields=['processed_on', 'process_count', 'process_log']
+        )
+
+    def undo(self, dryrun=True):
+        from scrobbles.tsv import undo_audioscrobbler_tsv_import
+
+        undo_audioscrobbler_tsv_import(self.process_log, dryrun)
+
+
+class LastFmImport(TimeStampedModel):
+    user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
+    uuid = models.UUIDField(editable=False, default=uuid4)
+    processing_started = models.DateTimeField(**BNULL)
+    processed_finished = models.DateTimeField(**BNULL)
+    process_log = models.TextField(**BNULL)
+    process_count = models.IntegerField(**BNULL)
+
+    def __str__(self):
+        return f"LastFM Import: {self.uuid}"
+
+    def process(self, import_all=False):
+        """Import scrobbles found on LastFM"""
+        if self.processed_finished:
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
+            return
+
+        last_import = None
+        if not import_all:
+            try:
+                last_import = LastFmImport.objects.exclude(id=self.id).last()
+            except:
+                pass
+
+        if not import_all and not last_import:
+            logger.warn(
+                "No previous import, to import all Last.fm scrobbles, pass import_all=True"
+            )
+            return
+
+        lastfm = LastFM(self.user)
+        last_processed = None
+        if last_import:
+            last_processed = last_import.processed_finished
+
+        self.processing_started = timezone.now()
+        self.save(update_fields=['processing_started'])
+
+        scrobbles = lastfm.import_from_lastfm(last_processed)
+        self.process_log = ""
+        if scrobbles:
+            for count, scrobble in enumerate(scrobbles):
+                scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}"
+                log_line = f"{scrobble_str}"
+                if count > 0:
+                    log_line = "\n" + log_line
+                self.process_log += log_line
+            self.process_count = len(scrobbles)
+        else:
+            self.process_count = 0
+
+        self.processed_finished = timezone.now()
+        self.save(
+            update_fields=[
+                'processed_finished',
+                'process_count',
+                'process_log',
+            ]
+        )
+
+    def undo(self, dryrun=False):
+        """Undo import of scrobbles from LastFM"""
+        LastFM.undo_lastfm_import(self.process_log, dryrun)
+        self.processed_finished = None
+        self.save(update_fields=['processed_finished'])
+
+
+class ChartRecord(TimeStampedModel):
+    """Sort of like a materialized view for what we could dynamically generate,
+    but would kill the DB as it gets larger. Collects time-based records
+    generated by a cron-like archival job
+
+    1972 by Josh Rouse - #3 in 2023, January
+
+    """
+
+    user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
+    rank = models.IntegerField()
+    year = models.IntegerField(default=timezone.now().year)
+    month = models.IntegerField(**BNULL)
+    week = models.IntegerField(**BNULL)
+    day = models.IntegerField(**BNULL)
+    video = models.ForeignKey(Video, on_delete=models.DO_NOTHING, **BNULL)
+    series = models.ForeignKey(Series, on_delete=models.DO_NOTHING, **BNULL)
+    artist = models.ForeignKey(Artist, on_delete=models.DO_NOTHING, **BNULL)
+    track = models.ForeignKey(Track, on_delete=models.DO_NOTHING, **BNULL)
+
+    @property
+    def media_obj(self):
+        media_obj = None
+        if self.video:
+            media_obj = self.video
+        if self.track:
+            media_obj = self.track
+        return media_obj
+
+    @property
+    def month_str(self) -> str:
+        month_str = ""
+        if self.month:
+            month_str = calendar.month_name[self.month]
+        return month_str
+
+    @property
+    def day_str(self) -> str:
+        day_str = ""
+        if self.day:
+            day_str = str(self.day)
+        return day_str
+
+    @property
+    def week_str(self) -> str:
+        week_str = ""
+        if self.week:
+            week_str = str(self.week)
+        return "Week " + week_str
+
+    @property
+    def period(self) -> str:
+        period = str(self.year)
+        if self.month:
+            period = " ".join([self.month_str, period])
+        if self.week:
+            period = " ".join([self.week_str, period])
+        if self.day:
+            period = " ".join([self.day_str, period])
+        return period
+
+    @property
+    def period_type(self) -> str:
+        period = 'year'
+        if self.month:
+            period = 'month'
+        if self.week:
+            period = 'week'
+        if self.day:
+            period = 'day'
+        return period
+
+    def __str__(self):
+        return f"#{self.rank} in {self.period} - {self.media_obj}"
 
 
 class Scrobble(TimeStampedModel):
+    """A scrobble tracks played media items by a user."""
+
+    uuid = models.UUIDField(editable=False, **BNULL)
     video = models.ForeignKey(Video, on_delete=models.DO_NOTHING, **BNULL)
     track = models.ForeignKey(Track, on_delete=models.DO_NOTHING, **BNULL)
     podcast_episode = models.ForeignKey(
         Episode, on_delete=models.DO_NOTHING, **BNULL
     )
+    sport_event = models.ForeignKey(
+        SportEvent, on_delete=models.DO_NOTHING, **BNULL
+    )
     user = models.ForeignKey(
         User, blank=True, null=True, on_delete=models.DO_NOTHING
     )
@@ -40,26 +244,59 @@ class Scrobble(TimeStampedModel):
     in_progress = models.BooleanField(default=True)
     scrobble_log = models.TextField(**BNULL)
 
+    def save(self, *args, **kwargs):
+        if not self.uuid:
+            self.uuid = uuid4()
+
+        return super(Scrobble, self).save(*args, **kwargs)
+
+    @property
+    def status(self) -> str:
+        if self.is_paused:
+            return 'paused'
+        if self.played_to_completion:
+            return 'finished'
+        if self.in_progress:
+            return 'in-progress'
+        return 'zombie'
+
+    @property
+    def is_stale(self) -> bool:
+        """Mark scrobble as stale if it's been more than an hour since it was updated"""
+        is_stale = False
+        now = timezone.now()
+        seconds_since_last_update = (now - self.modified).seconds
+        if seconds_since_last_update >= self.media_obj.SECONDS_TO_STALE:
+            is_stale = True
+        return is_stale
+
     @property
     def percent_played(self) -> int:
-        if (
-            self.playback_position_ticks
-            and self.media_obj.run_time_ticks
-            and self.source != 'Mopidy'
-        ):
-            return int(
-                (self.playback_position_ticks / self.media_obj.run_time_ticks)
-                * 100
-            )
-        # If we don't have media_obj.run_time_ticks, let's guess from created time
-        now = timezone.now()
-        playback_duration = (now - self.created).seconds
-        if playback_duration and self.media_obj.run_time:
-            return int(
-                (playback_duration / int(self.media_obj.run_time)) * 100
-            )
+        if not self.media_obj.run_time_ticks:
+            return 100
 
-        return 0
+        if not self.playback_position_ticks and self.played_to_completion:
+            return 100
+
+        playback_ticks = self.playback_position_ticks
+        if not playback_ticks:
+            playback_ticks = (timezone.now() - self.timestamp).seconds * 1000
+
+        percent = int((playback_ticks / self.media_obj.run_time_ticks) * 100)
+        if percent > 100:
+            percent = 100
+        return percent
+
+    @property
+    def can_be_updated(self) -> bool:
+        updatable = True
+        if self.percent_played > 100:
+            logger.info(f"No - 100% played - {self.id} - {self.source}")
+            updatable = False
+        if self.is_stale:
+            logger.info(f"No - stale - {self.id} - {self.source}")
+            updatable = False
+        return updatable
 
     @property
     def media_obj(self):
@@ -70,168 +307,126 @@ class Scrobble(TimeStampedModel):
             media_obj = self.track
         if self.podcast_episode:
             media_obj = self.podcast_episode
+        if self.sport_event:
+            media_obj = self.sport_event
         return media_obj
 
     def __str__(self):
-        return f"Scrobble of {self.media_obj} {self.timestamp.year}-{self.timestamp.month}"
+        timestamp = self.timestamp.strftime('%Y-%m-%d')
+        return f"Scrobble of {self.media_obj} ({timestamp})"
 
     @classmethod
-    def create_or_update_for_video(
-        cls, video: "Video", user_id: int, jellyfin_data: dict
+    def create_or_update(
+        cls, media, user_id: int, scrobble_data: dict
     ) -> "Scrobble":
-        jellyfin_data['video_id'] = video.id
-        logger.debug(
-            f"Creating or updating scrobble for video {video} with data {jellyfin_data}"
-        )
+
+        if media.__class__.__name__ == 'Track':
+            media_query = models.Q(track=media)
+            scrobble_data['track_id'] = media.id
+        if media.__class__.__name__ == 'Video':
+            media_query = models.Q(video=media)
+            scrobble_data['video_id'] = media.id
+        if media.__class__.__name__ == 'Episode':
+            media_query = models.Q(podcast_episode=media)
+            scrobble_data['podcast_episode_id'] = media.id
+        if media.__class__.__name__ == 'SportEvent':
+            media_query = models.Q(sport_event=media)
+            scrobble_data['sport_event_id'] = media.id
+
         scrobble = (
-            cls.objects.filter(video=video, user_id=user_id)
+            cls.objects.filter(
+                media_query,
+                user_id=user_id,
+            )
             .order_by('-modified')
             .first()
         )
+        if scrobble and scrobble.can_be_updated:
+            logger.info(
+                f"Updating {scrobble.id}",
+                {"scrobble_data": scrobble_data, "media": media},
+            )
+            return scrobble.update(scrobble_data)
 
-        # Backoff is how long until we consider this a new scrobble
-        backoff = timezone.now() + timedelta(minutes=VIDEO_BACKOFF)
-        wait_period = timezone.now() + timedelta(days=VIDEO_WAIT_PERIOD)
-
-        return cls.update_or_create(
-            scrobble, backoff, wait_period, jellyfin_data
+        source = scrobble_data['source']
+        logger.info(
+            f"Creating for {media.id} - {source}",
+            {"scrobble_data": scrobble_data, "media": media},
         )
+        # If creating a new scrobble, we don't need status
+        scrobble_data.pop('mopidy_status', None)
+        scrobble_data.pop('jellyfin_status', None)
+        return cls.create(scrobble_data)
 
-    @classmethod
-    def create_or_update_for_track(
-        cls, track: "Track", user_id: int, scrobble_data: dict
-    ) -> "Scrobble":
-        scrobble_data['track_id'] = track.id
-        scrobble = (
-            cls.objects.filter(track=track, user_id=user_id)
-            .order_by('-modified')
-            .first()
-        )
-        logger.debug(
-            f"Found existing scrobble for track {track}, updating",
-            {"scrobble_data": scrobble_data},
-        )
-
-        backoff = timezone.now() + timedelta(seconds=TRACK_BACKOFF)
-        wait_period = timezone.now() + timedelta(minutes=TRACK_WAIT_PERIOD)
-
-        return cls.update_or_create(
-            scrobble, backoff, wait_period, scrobble_data
-        )
-
-    @classmethod
-    def create_or_update_for_podcast_episode(
-        cls, episode: "Episode", user_id: int, scrobble_data: dict
-    ) -> "Scrobble":
-        scrobble_data['podcast_episode_id'] = episode.id
-        scrobble = (
-            cls.objects.filter(podcast_episode=episode, user_id=user_id)
-            .order_by('-modified')
-            .first()
-        )
-        logger.debug(
-            f"Found existing scrobble for podcast {episode}, updating",
-            {"scrobble_data": scrobble_data},
-        )
-
-        backoff = timezone.now() + timedelta(seconds=TRACK_BACKOFF)
-        wait_period = timezone.now() + timedelta(minutes=TRACK_WAIT_PERIOD)
-
-        return cls.update_or_create(
-            scrobble, backoff, wait_period, scrobble_data
-        )
-
-    @classmethod
-    def update_or_create(
-        cls,
-        scrobble: Optional["Scrobble"],
-        backoff,
-        wait_period,
-        scrobble_data: dict,
-    ) -> Optional["Scrobble"]:
-
+    def update(self, scrobble_data: dict) -> "Scrobble":
         # Status is a field we get from Mopidy, which refuses to poll us
         scrobble_status = scrobble_data.pop('mopidy_status', None)
         if not scrobble_status:
             scrobble_status = scrobble_data.pop('jellyfin_status', None)
-        if not scrobble_status:
-            logger.warning(
-                f"No status update found in message, not scrobbling"
-            )
-            return
 
-        logger.debug(f"Scrobbling to {scrobble} with status {scrobble_status}")
-        if scrobble:
-            scrobble.update_ticks(scrobble_data)
+        if self.percent_played < 100:
+            # Only worry about ticks if we haven't gotten to the end
+            self.update_ticks(scrobble_data)
 
-            # On stop, stop progress and send it to the check for completion
-            if scrobble_status == "stopped":
-                return scrobble.stop()
+        # On stop, stop progress and send it to the check for completion
+        if scrobble_status == "stopped":
+            self.stop()
+        # On pause, set is_paused and stop scrobbling
+        if scrobble_status == "paused":
+            self.pause()
+        if scrobble_status == "resumed":
+            self.resume()
 
-            # On pause, set is_paused and stop scrobbling
-            if scrobble_status == "paused":
-                return scrobble.pause()
-
-            if scrobble_status == "resumed":
-                return scrobble.resume()
-
-            # We're not changing the scrobble, but we don't want to walk over an existing one
-            scrobble_is_finished = (
-                not scrobble.in_progress and scrobble.modified < backoff
-            )
-            if scrobble_is_finished:
-                logger.info(
-                    'Found a very recent scrobble for this item, holding off scrobbling again'
-                )
-                return
-
-        if not scrobble:
-            # If we default this to "" we can probably remove this
-            scrobble_data['scrobble_log'] = ""
-            scrobble = cls.objects.create(
-                **scrobble_data,
-            )
-        else:
-            for key, value in scrobble_data.items():
-                setattr(scrobble, key, value)
-            scrobble.save()
-
-        # If we hit our completion threshold, save it and get ready
-        # to scrobble again if we re-watch this.
-        scrobble = check_scrobble_for_finish(scrobble)
+        for key, value in scrobble_data.items():
+            setattr(self, key, value)
+        self.save()
+        return self
 
+    @classmethod
+    def create(
+        cls,
+        scrobble_data: dict,
+    ) -> "Scrobble":
+        scrobble_data['scrobble_log'] = ""
+        scrobble = cls.objects.create(
+            **scrobble_data,
+        )
         return scrobble
 
-    def stop(self):
+    def stop(self, force_finish=False) -> None:
         if not self.in_progress:
-            logger.warning("Scrobble already stopped")
             return
         self.in_progress = False
         self.save(update_fields=['in_progress'])
-        return check_scrobble_for_finish(self)
+        logger.info(f"{self.id} - {self.source}")
+        check_scrobble_for_finish(self, force_finish)
 
-    def pause(self):
+    def pause(self) -> None:
         if self.is_paused:
-            logger.warning("Scrobble already paused")
+            logger.warning(f"{self.id} - already paused - {self.source}")
             return
         self.is_paused = True
         self.save(update_fields=["is_paused"])
-        return check_scrobble_for_finish(self)
+        logger.info(f"{self.id} - pausing - {self.source}")
+        check_scrobble_for_finish(self)
 
-    def resume(self):
+    def resume(self) -> None:
         if self.is_paused or not self.in_progress:
             self.is_paused = False
             self.in_progress = True
+            logger.info(f"{self.id} - resuming - {self.source}")
             return self.save(update_fields=["is_paused", "in_progress"])
-        return self
 
-    def update_ticks(self, data):
+    def cancel(self) -> None:
+        check_scrobble_for_finish(self, force_finish=True)
+        self.delete()
+
+    def update_ticks(self, data) -> None:
         self.playback_position_ticks = data.get("playback_position_ticks")
         self.playback_position = data.get("playback_position")
-        logger.debug(
-            f"Updating scrobble ticks to {self.playback_position_ticks}"
+        logger.info(
+            f"{self.id} - {self.playback_position_ticks} - {self.source}"
         )
         self.save(
             update_fields=['playback_position_ticks', 'playback_position']
         )
-        return self

vrobbler/apps/scrobbles/musicbrainz.py

@@ -0,0 +1,107 @@
+import logging
+
+import musicbrainzngs
+from dateutil.parser import parse
+
+logger = logging.getLogger(__name__)
+
+
+def lookup_album_from_mb(musicbrainz_id: str) -> dict:
+    release_dict = {}
+
+    musicbrainzngs.set_useragent('vrobbler', '0.3.0')
+    release_data = musicbrainzngs.get_release_by_id(
+        musicbrainz_id,
+        includes=['artists', 'release-groups', 'recordings'],
+    ).get('release')
+
+    if not release_data:
+        return release_dict
+
+    primary_artist = release_data.get('artist-credit')[0]
+    release_dict = {
+        'artist': {
+            'name': primary_artist.get('name'),
+            'musicbrainz_id': primary_artist.get('id'),
+        },
+        'album': {
+            'name': release_data.get('title'),
+            'musicbrainz_id': musicbrainz_id,
+            'musicbrainz_releasegroup_id': release_data.get(
+                'release-group'
+            ).get('id'),
+            'musicbrainz_albumaritist_id': primary_artist.get('id'),
+            'year': release_data.get('year')[0:4],
+        },
+    }
+
+    release_dict['tracks'] = []
+    for track in release_data.get('medium-list')[0]['track-list']:
+        recording = track['recording']
+        release_dict['tracks'].append(
+            {
+                'title': recording['title'],
+                'musicbrainz_id': recording['id'],
+                'run_time_ticks': track['length'],
+            }
+        )
+
+    return release_dict
+
+
+def lookup_album_dict_from_mb(release_name: str, artist_name: str) -> dict:
+    musicbrainzngs.set_useragent('vrobbler', '0.3.0')
+
+    top_result = musicbrainzngs.search_releases(
+        release_name, artist=artist_name
+    )['release-list'][0]
+    score = int(top_result.get('ext:score'))
+    if score < 85:
+        logger.debug(
+            "Album lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return {}
+    year = None
+    if top_result.get("date"):
+        year = parse(top_result["date"]).year
+
+    return {
+        "year": year,
+        "mb_id": top_result["id"],
+        "mb_group_id": top_result["release-group"]["id"],
+    }
+
+
+def lookup_artist_from_mb(artist_name: str) -> str:
+    musicbrainzngs.set_useragent('vrobbler', '0.3.0')
+
+    top_result = musicbrainzngs.search_artists(artist=artist_name)[
+        'artist-list'
+    ][0]
+    score = int(top_result.get('ext:score'))
+    if score < 85:
+        logger.debug(
+            "Artist lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return ""
+
+    return top_result
+
+
+def lookup_track_from_mb(artist_name: str) -> str:
+    musicbrainzngs.set_useragent('vrobbler', '0.3.0')
+
+    top_result = musicbrainzngs.search_recordings(artist=artist_name)[
+        'artist-list'
+    ][0]
+    score = int(top_result.get('ext:score'))
+    if score < 85:
+        logger.debug(
+            "Artist lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return ""
+
+    return top_result

vrobbler/apps/scrobbles/scrobblers.py

@@ -9,6 +9,12 @@ from podcasts.models import Episode
 from scrobbles.models import Scrobble
 from scrobbles.utils import convert_to_seconds, parse_mopidy_uri
 from videos.models import Video
+from sports.models import SportEvent
+from vrobbler.apps.music.utils import (
+    get_or_create_album,
+    get_or_create_artist,
+    get_or_create_track,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -49,32 +55,30 @@ def mopidy_scrobble_podcast(
 
     scrobble = None
     if episode:
-        scrobble = Scrobble.create_or_update_for_podcast_episode(
-            episode, user_id, mopidy_data
-        )
+        scrobble = Scrobble.create_or_update(episode, user_id, mopidy_data)
     return scrobble
 
 
 def mopidy_scrobble_track(
     data_dict: dict, user_id: Optional[int]
 ) -> Optional[Scrobble]:
-    artist_dict = {
-        "name": data_dict.get("artist", None),
-        "musicbrainz_id": data_dict.get("musicbrainz_artist_id", None),
-    }
-
-    album_dict = {
-        "name": data_dict.get("album"),
-        "musicbrainz_id": data_dict.get("musicbrainz_album_id"),
-    }
-
-    track_dict = {
-        "title": data_dict.get("name"),
-        "run_time_ticks": data_dict.get("run_time_ticks"),
-        "run_time": data_dict.get("run_time"),
-    }
-
-    track = Track.find_or_create(artist_dict, album_dict, track_dict)
+    artist = get_or_create_artist(
+        data_dict.get("artist"),
+        mbid=data_dict.get("musicbrainz_artist_id", None),
+    )
+    album = get_or_create_album(
+        data_dict.get("album"),
+        artist=artist,
+        mbid=data_dict.get("musicbrainz_album_id"),
+    )
+    track = get_or_create_track(
+        title=data_dict.get("name"),
+        mbid=data_dict.get("musicbrainz_track_id"),
+        artist=artist,
+        album=album,
+        run_time_ticks=data_dict.get("run_time_ticks"),
+        run_time=data_dict.get("run_time"),
+    )
 
     # Now we run off a scrobble
     mopidy_data = {
@@ -85,34 +89,32 @@ def mopidy_scrobble_track(
         "mopidy_status": data_dict.get("status"),
     }
 
-    scrobble = None
+    # Jellyfin MB ids suck, so always overwrite with Mopidy if they're offering
+    track.musicbrainz_id = data_dict.get("musicbrainz_track_id")
+    track.save()
+
+    scrobble = Scrobble.create_or_update(track, user_id, mopidy_data)
 
-    if track:
-        # Jellyfin MB ids suck, so always overwrite with Mopidy if they're offering
-        track.musicbrainz_id = data_dict.get("musicbrainz_track_id")
-        track.save()
-        scrobble = Scrobble.create_or_update_for_track(
-            track, user_id, mopidy_data
-        )
     return scrobble
 
 
-def create_jellyfin_scrobble_dict(data_dict: dict, user_id: int) -> dict:
+def build_scrobble_dict(data_dict: dict, user_id: int) -> dict:
     jellyfin_status = "resumed"
     if data_dict.get("IsPaused"):
         jellyfin_status = "paused"
-    if data_dict.get("PlayedToCompletion"):
+    elif data_dict.get("NotificationType") == 'PlaybackStop':
         jellyfin_status = "stopped"
 
+    playback_ticks = data_dict.get("PlaybackPositionTicks", "")
+    if playback_ticks:
+        playback_ticks = playback_ticks // 10000
+
     return {
         "user_id": user_id,
         "timestamp": parse(data_dict.get("UtcTimestamp")),
-        "playback_position_ticks": data_dict.get("PlaybackPositionTicks")
-        // 10000,
-        "playback_position": convert_to_seconds(
-            data_dict.get("PlaybackPosition")
-        ),
-        "source": "Jellyfin",
+        "playback_position_ticks": playback_ticks,
+        "playback_position": data_dict.get("PlaybackPosition", ""),
+        "source": data_dict.get("ClientName", "Vrobbler"),
         "source_id": data_dict.get('MediaSourceId'),
         "jellyfin_status": jellyfin_status,
     }
@@ -121,48 +123,49 @@ def create_jellyfin_scrobble_dict(data_dict: dict, user_id: int) -> dict:
 def jellyfin_scrobble_track(
     data_dict: dict, user_id: Optional[int]
 ) -> Optional[Scrobble]:
-    if not data_dict.get("Provider_musicbrainztrack", None):
-        logger.error(
-            "No MBrainz Track ID received. This is likely because all metadata is bad, not scrobbling"
-        )
+
+    null_position_on_progress = (
+        data_dict.get("PlaybackPosition") == "00:00:00"
+        and data_dict.get("NotificationType") == "PlaybackProgress"
+    )
+
+    # Jellyfin has some race conditions with it's webhooks, these hacks fix some of them
+    if not data_dict.get("PlaybackPositionTicks") or null_position_on_progress:
+        logger.error("No playback position tick from Jellyfin, aborting")
         return
 
-    artist_dict = {
-        'name': data_dict.get(JELLYFIN_POST_KEYS["ARTIST_NAME"], None),
-        'musicbrainz_id': data_dict.get(
-            JELLYFIN_POST_KEYS["ARTIST_MB_ID"], None
-        ),
-    }
+    artist = get_or_create_artist(
+        data_dict.get(JELLYFIN_POST_KEYS["ARTIST_NAME"]),
+        mbid=data_dict.get(JELLYFIN_POST_KEYS["ARTIST_MB_ID"]),
+    )
+    album = get_or_create_album(
+        data_dict.get(JELLYFIN_POST_KEYS["ALBUM_NAME"]),
+        artist=artist,
+        mbid=data_dict.get(JELLYFIN_POST_KEYS['ALBUM_MB_ID']),
+    )
 
-    album_dict = {
-        "name": data_dict.get(JELLYFIN_POST_KEYS["ALBUM_NAME"], None),
-        "musicbrainz_id": data_dict.get(JELLYFIN_POST_KEYS['ALBUM_MB_ID']),
-    }
+    run_time_ticks = (
+        data_dict.get(JELLYFIN_POST_KEYS["RUN_TIME_TICKS"]) // 10000
+    )
+    run_time = convert_to_seconds(
+        data_dict.get(JELLYFIN_POST_KEYS["RUN_TIME"])
+    )
+    track = get_or_create_track(
+        title=data_dict.get("Name"),
+        mbid=data_dict.get(JELLYFIN_POST_KEYS["TRACK_MB_ID"]),
+        artist=artist,
+        album=album,
+        run_time_ticks=run_time_ticks,
+        run_time=run_time,
+    )
 
-    # Convert ticks from Jellyfin from microseconds to nanoseconds
-    # Ain't nobody got time for nanoseconds
-    track_dict = {
-        "title": data_dict.get("Name", ""),
-        "run_time_ticks": data_dict.get(
-            JELLYFIN_POST_KEYS["RUN_TIME_TICKS"], None
-        )
-        // 10000,
-        "run_time": convert_to_seconds(
-            data_dict.get(JELLYFIN_POST_KEYS["RUN_TIME"], None)
-        ),
-    }
-    track = Track.find_or_create(artist_dict, album_dict, track_dict)
+    scrobble_dict = build_scrobble_dict(data_dict, user_id)
 
-    # Prefer Mopidy MD IDs to Jellyfin, so skip if we already have one
-    if not track.musicbrainz_id:
-        track.musicbrainz_id = data_dict.get(
-            JELLYFIN_POST_KEYS["TRACK_MB_ID"], None
-        )
-        track.save()
+    # A hack to make Jellyfin work more like Mopidy for music tracks
+    scrobble_dict["playback_position_ticks"] = 0
+    scrobble_dict["playback_position"] = ""
 
-    scrobble_dict = create_jellyfin_scrobble_dict(data_dict, user_id)
-
-    return Scrobble.create_or_update_for_track(track, user_id, scrobble_dict)
+    return Scrobble.create_or_update(track, user_id, scrobble_dict)
 
 
 def jellyfin_scrobble_video(data_dict: dict, user_id: Optional[int]):
@@ -173,6 +176,32 @@ def jellyfin_scrobble_video(data_dict: dict, user_id: Optional[int]):
         return
     video = Video.find_or_create(data_dict)
 
-    scrobble_dict = create_jellyfin_scrobble_dict(data_dict, user_id)
+    scrobble_dict = build_scrobble_dict(data_dict, user_id)
 
-    return Scrobble.create_or_update_for_video(video, user_id, scrobble_dict)
+    return Scrobble.create_or_update(video, user_id, scrobble_dict)
+
+
+def manual_scrobble_video(data_dict: dict, user_id: Optional[int]):
+    if not data_dict.get("Provider_imdb", None):
+        logger.error(
+            "No IMDB ID received. This is likely because all metadata is bad, not scrobbling"
+        )
+        return
+    video = Video.find_or_create(data_dict)
+
+    scrobble_dict = build_scrobble_dict(data_dict, user_id)
+
+    return Scrobble.create_or_update(video, user_id, scrobble_dict)
+
+
+def manual_scrobble_event(data_dict: dict, user_id: Optional[int]):
+    if not data_dict.get("Provider_thesportsdb", None):
+        logger.error(
+            "No TheSportsDB ID received. This is likely because all metadata is bad, not scrobbling"
+        )
+        return
+    event = SportEvent.find_or_create(data_dict)
+
+    scrobble_dict = build_scrobble_dict(data_dict, user_id)
+
+    return Scrobble.create_or_update(event, user_id, scrobble_dict)

vrobbler/apps/scrobbles/serializers.py

@@ -1,5 +1,11 @@
 from rest_framework import serializers
-from scrobbles.models import Scrobble
+from scrobbles.models import Scrobble, AudioScrobblerTSVImport
+
+
+class AudioScrobblerTSVImportSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = AudioScrobblerTSVImport
+        fields = ('tsv_file',)
 
 
 class ScrobbleSerializer(serializers.ModelSerializer):

vrobbler/apps/scrobbles/stats.py

@@ -0,0 +1,115 @@
+import calendar
+import logging
+from datetime import datetime, timedelta
+from typing import Optional
+
+import pytz
+from django.apps import apps
+from django.conf import settings
+from django.db.models import Count, Q
+
+
+logger = logging.getLogger(__name__)
+
+
+def get_start_end_dates_by_week(year, week, tz):
+    d = datetime(year, 1, 1, tzinfo=tz)
+    if d.weekday() <= 3:
+        d = d - timedelta(d.weekday())
+    else:
+        d = d + timedelta(7 - d.weekday())
+    dlt = timedelta(days=(week - 1) * 7)
+    return d + dlt, d + dlt + timedelta(days=6)
+
+
+def get_scrobble_count_qs(
+    year: Optional[int] = None,
+    month: Optional[int] = None,
+    week: Optional[int] = None,
+    day: Optional[int] = None,
+    user=None,
+    model_str="Track",
+) -> dict[str, int]:
+
+    tz = settings.TIME_ZONE
+    if user and user.is_authenticated:
+        tz = pytz.timezone(user.profile.timezone)
+
+    data_model = apps.get_model(app_label='music', model_name='Track')
+    if model_str == "Video":
+        data_model = apps.get_model(app_label='videos', model_name='Video')
+    if model_str == "SportEvent":
+        data_model = apps.get_model(
+            app_label='sports', model_name='SportEvent'
+        )
+
+    base_qs = data_model.objects.filter(
+        scrobble__user=user,
+        scrobble__played_to_completion=True,
+    )
+
+    # Returna all media items with scrobble count annotated
+    if not year:
+        return base_qs.annotate(scrobble_count=Count("scrobble")).order_by(
+            "-scrobble_count"
+        )
+
+    start = datetime(year, 1, 1, tzinfo=tz)
+    end = datetime(year, 12, 31, tzinfo=tz)
+    if month:
+        end_day = calendar.monthrange(year, month)[1]
+        start = datetime(year, month, 1, tzinfo=tz)
+        end = datetime(year, month, end_day, tzinfo=tz)
+    elif week:
+        start, end = get_start_end_dates_by_week(year, week, tz)
+    elif day and month:
+        start = datetime(year, month, day, 0, 0, tzinfo=tz)
+        end = datetime(year, month, day, 23, 59, tzinfo=tz)
+    elif day and not month:
+        logger.warning('Day provided with month, defaulting ot all-time')
+
+    date_filter = Q(
+        scrobble__timestamp__gte=start, scrobble__timestamp__lte=end
+    )
+
+    return (
+        base_qs.annotate(
+            scrobble_count=Count("scrobble", filter=Q(date_filter))
+        )
+        .filter(date_filter)
+        .order_by("-scrobble_count")
+    )
+
+
+def build_charts(
+    year: Optional[int] = None,
+    month: Optional[int] = None,
+    week: Optional[int] = None,
+    day: Optional[int] = None,
+    user=None,
+    model_str="Track",
+):
+    ChartRecord = apps.get_model(
+        app_label='scrobbles', model_name='ChartRecord'
+    )
+    results = get_scrobble_count_qs(year, month, week, day, user, model_str)
+    unique_counts = list(set([result.scrobble_count for result in results]))
+    unique_counts.sort(reverse=True)
+    ranks = {}
+    for rank, count in enumerate(unique_counts, start=1):
+        ranks[count] = rank
+
+    chart_records = []
+    for result in results:
+        chart_record = {
+            'year': year,
+            'week': week,
+            'month': month,
+            'day': day,
+            'user': user,
+        }
+        chart_record['rank'] = ranks[result.scrobble_count]
+        if model_str == 'Track':
+            chart_record['track'] = result
+        chart_records.append(ChartRecord(**chart_record))
+    ChartRecord.objects.bulk_create(chart_records, ignore_conflicts=True)

vrobbler/apps/scrobbles/tasks.py

@@ -0,0 +1,24 @@
+import logging
+from celery import shared_task
+
+from scrobbles.models import AudioScrobblerTSVImport, LastFmImport
+
+logger = logging.getLogger(__name__)
+
+
+@shared_task
+def process_lastfm_import(import_id):
+    lastfm_import = LastFmImport.objects.filter(id=import_id).first()
+    if not lastfm_import:
+        logger.warn(f"LastFmImport not found with id {import_id}")
+
+    lastfm_import.process()
+
+
+@shared_task
+def process_tsv_import(import_id):
+    tsv_import = AudioScrobblerTSVImport.objects.filter(id=import_id).first()
+    if not tsv_import:
+        logger.warn(f"AudioScrobblerTSVImport not found with id {import_id}")
+
+    tsv_import.process()

vrobbler/apps/scrobbles/thesportsdb.py

@@ -0,0 +1,51 @@
+import logging
+
+from dateutil.parser import parse
+from django.conf import settings
+from django.utils import timezone
+from pysportsdb import TheSportsDbClient
+from sports.models import Sport
+
+logger = logging.getLogger(__name__)
+
+API_KEY = getattr(settings, "THESPORTSDB_API_KEY", "2")
+client = TheSportsDbClient(api_key=API_KEY)
+
+
+def lookup_event_from_thesportsdb(event_id: str) -> dict:
+
+    event = client.lookup_event(event_id)['events'][0]
+    if not event or type(event) != dict:
+        return {}
+    league = {}  # client.lookup_league(league_id=event.get('idLeague'))
+    event_type = "Game"
+    sport, _created = Sport.objects.get_or_create(
+        thesportsdb_id=event.get('strSport')
+    )
+
+    data_dict = {
+        "ItemType": sport.default_event_type,
+        "Name": event.get('strEvent'),
+        "AltName": event.get('strEventAlternate'),
+        "Start": parse(event.get('strTimestamp')),
+        "Provider_thesportsdb": event.get('idEvent'),
+        "RunTime": sport.default_event_run_time,
+        "RunTimeTicks": sport.default_event_run_time_ticks,
+        "Sport": event.get('strSport'),
+        "Season": event.get('strSeason'),
+        "LeagueId": event.get('idLeague'),
+        "LeagueName": event.get('strLeague'),
+        "HomeTeamId": event.get('idHomeTeam'),
+        "HomeTeamName": event.get('strHomeTeam'),
+        "AwayTeamId": event.get('idAwayTeam'),
+        "AwayTeamName": event.get('strAwayTeam'),
+        "RoundId": event.get('intRound'),
+        "PlaybackPositionTicks": None,
+        "PlaybackPosition": None,
+        "UtcTimestamp": timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f%z'),
+        "IsPaused": False,
+        "PlayedToCompletion": False,
+        "Source": "Vrobbler",
+    }
+
+    return data_dict

vrobbler/apps/scrobbles/tsv.py

@@ -0,0 +1,96 @@
+import csv
+import logging
+from datetime import datetime
+
+import pytz
+from scrobbles.models import Scrobble
+from music.utils import (
+    get_or_create_album,
+    get_or_create_artist,
+    get_or_create_track,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def process_audioscrobbler_tsv_file(file_path, user_tz=None):
+    """Takes a path to a file of TSV data and imports it as past scrobbles"""
+    new_scrobbles = []
+    if not user_tz:
+        user_tz = pytz.utc
+
+    with open(file_path) as infile:
+        source = 'Audioscrobbler File'
+        rows = csv.reader(infile, delimiter="\t")
+
+        source_id = ""
+        for row_num, row in enumerate(rows):
+            if row_num in [0, 1, 2]:
+                if "Rockbox" in row[0]:
+                    source = "Rockbox"
+                source_id += row[0] + "\n"
+                continue
+            if len(row) > 8:
+                logger.warning(
+                    'Improper row length during Audioscrobbler import',
+                    extra={'row': row},
+                )
+                continue
+            artist = get_or_create_artist(row[0])
+            album = get_or_create_album(row[1], artist)
+
+            track = get_or_create_track(
+                title=row[2],
+                mbid=row[7],
+                artist=artist,
+                album=album,
+                run_time=row[4],
+                run_time_ticks=int(row[4]) * 1000,
+            )
+
+            timestamp = (
+                datetime.fromtimestamp(int(row[6]))
+                .replace(tzinfo=user_tz)
+                .astimezone(pytz.utc)
+            )
+
+            new_scrobble = Scrobble(
+                timestamp=timestamp,
+                source=source,
+                source_id=source_id,
+                track=track,
+                played_to_completion=True,
+                in_progress=False,
+            )
+            existing = Scrobble.objects.filter(
+                timestamp=timestamp, track=track
+            ).first()
+            if existing:
+                logger.debug(f"Skipping existing scrobble {new_scrobble}")
+                continue
+            logger.debug(f"Queued scrobble {new_scrobble} for creation")
+            new_scrobbles.append(new_scrobble)
+
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={'created_scrobbles': created},
+        )
+        return created
+
+
+def undo_audioscrobbler_tsv_import(process_log, dryrun=True):
+    """Accepts the log from a TSV import and removes the scrobbles"""
+    if not process_log:
+        logger.warning("No lines in process log found to undo")
+        return
+
+    for line in process_log.split('\n'):
+        scrobble_id = line.split("\t")[0]
+        scrobble = Scrobble.objects.filter(id=scrobble_id).first()
+        if not scrobble:
+            logger.warning(f"Could not find scrobble {scrobble_id} to undo")
+            continue
+        logger.info(f"Removing scrobble {scrobble_id}")
+        if not dryrun:
+            scrobble.delete()

vrobbler/apps/scrobbles/urls.py

@@ -4,7 +4,16 @@ from scrobbles import views
 app_name = 'scrobbles'
 
 urlpatterns = [
-    path('', views.scrobble_endpoint, name='scrobble-list'),
+    path('', views.scrobble_endpoint, name='api-list'),
+    path('finish/<slug:uuid>', views.scrobble_finish, name='finish'),
+    path('cancel/<slug:uuid>', views.scrobble_cancel, name='cancel'),
+    path(
+        'upload/',
+        views.AudioScrobblerImportCreateView.as_view(),
+        name='audioscrobbler-file-upload',
+    ),
+    path('lastfm-import/', views.lastfm_import, name='lastfm-import'),
     path('jellyfin/', views.jellyfin_websocket, name='jellyfin-websocket'),
     path('mopidy/', views.mopidy_websocket, name='mopidy-websocket'),
+    path('export/', views.export, name='export'),
 ]

vrobbler/apps/scrobbles/utils.py

@@ -1,5 +1,4 @@
 import logging
-from typing import Any, Optional
 from urllib.parse import unquote
 
 from dateutil.parser import ParserError, parse
@@ -10,10 +9,17 @@ logger = logging.getLogger(__name__)
 
 def convert_to_seconds(run_time: str) -> int:
     """Jellyfin sends run time as 00:00:00 string. We want the run time to
-    actually be in seconds so we'll convert it"""
-    if ":" in run_time:
+    actually be in seconds so we'll convert it"
+
+    This is actually deprecated, as we now convert to seconds before saving.
+    But for older videos, we'll leave this here.
+    """
+    if ":" in str(run_time):
         run_time_list = run_time.split(":")
-        run_time = (int(run_time_list[1]) * 60) + int(run_time_list[2])
+        hours = int(run_time_list[0])
+        minutes = int(run_time_list[1])
+        seconds = int(run_time_list[2])
+        run_time = (((hours * 60) + minutes) * 60) + seconds
     return int(run_time)
 
 
@@ -60,25 +66,40 @@ def parse_mopidy_uri(uri: str) -> dict:
     }
 
 
-def check_scrobble_for_finish(scrobble: "Scrobble") -> None:
-    completion_percent = getattr(settings, "MUSIC_COMPLETION_PERCENT", 95)
-    if scrobble.video:
-        completion_percent = getattr(settings, "VIDEO_COMPLETION_PERCENT", 90)
-    if scrobble.podcast_episode:
-        completion_percent = getattr(
-            settings, "PODCAST_COMPLETION_PERCENT", 25
-        )
-    if scrobble.percent_played >= completion_percent:
+def check_scrobble_for_finish(
+    scrobble: "Scrobble", force_to_100=False, force_finish=False
+) -> None:
+    completion_percent = scrobble.media_obj.COMPLETION_PERCENT
+
+    if scrobble.percent_played >= completion_percent or force_finish:
+        logger.info(f"{scrobble.id} {completion_percent} met, finishing")
+
+        if (
+            scrobble.playback_position_ticks
+            and scrobble.media_obj.run_time_ticks
+            and force_to_100
+        ):
+            scrobble.playback_position_ticks = (
+                scrobble.media_obj.run_time_ticks
+            )
+            logger.info(
+                f"{scrobble.playback_position_ticks} set to {scrobble.media_obj.run_time_ticks}"
+            )
+
         scrobble.in_progress = False
         scrobble.is_paused = False
         scrobble.played_to_completion = True
+
         scrobble.save(
-            update_fields=["in_progress", "is_paused", "played_to_completion"]
+            update_fields=[
+                "in_progress",
+                "is_paused",
+                "played_to_completion",
+                'playback_position_ticks',
+            ]
         )
 
     if scrobble.percent_played % 5 == 0:
         if getattr(settings, "KEEP_DETAILED_SCROBBLE_LOGS", False):
             scrobble.scrobble_log += f"\n{str(scrobble.timestamp)} - {scrobble.playback_position} - {str(scrobble.playback_position_ticks)} - {str(scrobble.percent_played)}%"
             scrobble.save(update_fields=['scrobble_log'])
-
-    return scrobble

vrobbler/apps/scrobbles/views.py

@@ -1,85 +1,105 @@
 import json
 import logging
-from datetime import datetime, timedelta
+from datetime import datetime
 
-from dateutil.parser import parse
+import pytz
 from django.conf import settings
+from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db.models.fields import timezone
+from django.http import FileResponse, HttpResponseRedirect, JsonResponse
+from django.urls import reverse, reverse_lazy
 from django.utils import timezone
 from django.views.decorators.csrf import csrf_exempt
+from django.views.generic import FormView
+from django.views.generic.edit import CreateView
 from django.views.generic.list import ListView
-from music.constants import JELLYFIN_POST_KEYS as KEYS
-from music.models import Track
-from rest_framework import status
-from rest_framework.decorators import api_view
-from rest_framework.response import Response
-from scrobbles.constants import (
-    JELLYFIN_AUDIO_ITEM_TYPES,
-    JELLYFIN_VIDEO_ITEM_TYPES,
-)
-from scrobbles.models import Scrobble
-from scrobbles.scrobblers import (
-    jellyfin_scrobble_track,
-    jellyfin_scrobble_video,
-    mopidy_scrobble_podcast,
-    mopidy_scrobble_track,
-)
-from scrobbles.serializers import ScrobbleSerializer
-from scrobbles.utils import convert_to_seconds
-from videos.models import Video
-
-from vrobbler.apps.music.aggregators import (
+from music.aggregators import (
     scrobble_counts,
     top_artists,
     top_tracks,
     week_of_scrobbles,
 )
+from rest_framework import status
+from rest_framework.decorators import (
+    api_view,
+    parser_classes,
+    permission_classes,
+)
+from rest_framework.parsers import MultiPartParser
+from rest_framework.permissions import IsAuthenticated
+from rest_framework.response import Response
+from scrobbles.constants import (
+    JELLYFIN_AUDIO_ITEM_TYPES,
+    JELLYFIN_VIDEO_ITEM_TYPES,
+)
+from scrobbles.export import export_scrobbles
+from scrobbles.forms import ExportScrobbleForm, ScrobbleForm
+from scrobbles.imdb import lookup_video_from_imdb
+from scrobbles.models import AudioScrobblerTSVImport, LastFmImport, Scrobble
+from scrobbles.scrobblers import (
+    jellyfin_scrobble_track,
+    jellyfin_scrobble_video,
+    manual_scrobble_event,
+    manual_scrobble_video,
+    mopidy_scrobble_podcast,
+    mopidy_scrobble_track,
+)
+from scrobbles.serializers import (
+    AudioScrobblerTSVImportSerializer,
+    ScrobbleSerializer,
+)
+from scrobbles.tasks import process_lastfm_import, process_tsv_import
+from scrobbles.thesportsdb import lookup_event_from_thesportsdb
 
 logger = logging.getLogger(__name__)
 
-TRUTHY_VALUES = [
-    'true',
-    '1',
-    't',
-    'y',
-    'yes',
-    'yeah',
-    'yup',
-    'certainly',
-    'uh-huh',
-]
-
 
 class RecentScrobbleList(ListView):
     model = Scrobble
 
     def get_context_data(self, **kwargs):
         data = super().get_context_data(**kwargs)
+        user = self.request.user
         now = timezone.now()
-        last_eight_minutes = timezone.now() - timedelta(minutes=8)
-        # Find scrobbles from the last 10 minutes
-        data['now_playing_list'] = Scrobble.objects.filter(
-            in_progress=True,
-            is_paused=False,
-            modified__gte=last_eight_minutes,
-            timestamp__lte=now,
-        )
-        data['video_scrobble_list'] = Scrobble.objects.filter(
-            video__isnull=False, played_to_completion=True
-        ).order_by('-timestamp')[:10]
-        data['podcast_scrobble_list'] = Scrobble.objects.filter(
-            podcast_episode__isnull=False, played_to_completion=True
-        ).order_by('-timestamp')[:10]
-        data['top_daily_tracks'] = top_tracks()
-        data['top_weekly_tracks'] = top_tracks(filter='week')
-        data['top_monthly_tracks'] = top_tracks(filter='month')
+        if user.is_authenticated:
+            if user.profile:
+                timezone.activate(pytz.timezone(user.profile.timezone))
+                now = timezone.localtime(timezone.now())
+            data['now_playing_list'] = Scrobble.objects.filter(
+                in_progress=True,
+                is_paused=False,
+                timestamp__lte=now,
+                user=user,
+            )
 
-        data['top_daily_artists'] = top_artists()
-        data['top_weekly_artists'] = top_artists(filter='week')
-        data['top_monthly_artists'] = top_artists(filter='month')
+            completed_for_user = Scrobble.objects.filter(
+                played_to_completion=True, user=user
+            )
+            data['video_scrobble_list'] = completed_for_user.filter(
+                video__isnull=False
+            ).order_by('-timestamp')[:15]
 
-        data["weekly_data"] = week_of_scrobbles()
-        data['counts'] = scrobble_counts()
+            data['podcast_scrobble_list'] = completed_for_user.filter(
+                podcast_episode__isnull=False
+            ).order_by('-timestamp')[:15]
+
+            data['sport_scrobble_list'] = completed_for_user.filter(
+                sport_event__isnull=False
+            ).order_by('-timestamp')[:15]
+
+            # data['top_daily_tracks'] = top_tracks()
+            data['top_weekly_tracks'] = top_tracks(user, filter='week')
+            data['top_monthly_tracks'] = top_tracks(user, filter='month')
+
+            # data['top_daily_artists'] = top_artists()
+            data['top_weekly_artists'] = top_artists(user, filter='week')
+            data['top_monthly_artists'] = top_artists(user, filter='month')
+
+        data["weekly_data"] = week_of_scrobbles(user=user)
+
+        data['counts'] = scrobble_counts(user)
+        data['imdb_form'] = ScrobbleForm
+        data['export_form'] = ExportScrobbleForm
         return data
 
     def get_queryset(self):
@@ -88,6 +108,88 @@ class RecentScrobbleList(ListView):
         ).order_by('-timestamp')[:15]
 
 
+class ManualScrobbleView(FormView):
+    form_class = ScrobbleForm
+    template_name = 'scrobbles/manual_form.html'
+
+    def form_valid(self, form):
+
+        item_id = form.cleaned_data.get('item_id')
+        data_dict = None
+        if 'tt' in item_id:
+            data_dict = lookup_video_from_imdb(
+                form.cleaned_data.get('item_id')
+            )
+            if data_dict:
+                manual_scrobble_video(data_dict, self.request.user.id)
+
+        if not data_dict:
+            logger.debug(f"Looking for sport event with ID {item_id}")
+            data_dict = lookup_event_from_thesportsdb(
+                form.cleaned_data.get('item_id')
+            )
+            if data_dict:
+                manual_scrobble_event(data_dict, self.request.user.id)
+
+        return HttpResponseRedirect(reverse("vrobbler-home"))
+
+
+class JsonableResponseMixin:
+    """
+    Mixin to add JSON support to a form.
+    Must be used with an object-based FormView (e.g. CreateView)
+    """
+
+    def form_invalid(self, form):
+        response = super().form_invalid(form)
+        if self.request.accepts('text/html'):
+            return response
+        else:
+            return JsonResponse(form.errors, status=400)
+
+    def form_valid(self, form):
+        # We make sure to call the parent's form_valid() method because
+        # it might do some processing (in the case of CreateView, it will
+        # call form.save() for example).
+        response = super().form_valid(form)
+        if self.request.accepts('text/html'):
+            return response
+        else:
+            data = {
+                'pk': self.object.pk,
+            }
+            return JsonResponse(data)
+
+
+class AudioScrobblerImportCreateView(
+    LoginRequiredMixin, JsonableResponseMixin, CreateView
+):
+    model = AudioScrobblerTSVImport
+    fields = ['tsv_file']
+    template_name = 'scrobbles/upload_form.html'
+    success_url = reverse_lazy('vrobbler-home')
+
+    def form_valid(self, form):
+        self.object = form.save(commit=False)
+        self.object.user = self.request.user
+        self.object.save()
+        process_tsv_import.delay(self.object.id)
+        return HttpResponseRedirect(self.get_success_url())
+
+
+@permission_classes([IsAuthenticated])
+@api_view(['GET'])
+def lastfm_import(request):
+    lfm_import, created = LastFmImport.objects.get_or_create(
+        user=request.user, processed_finished__isnull=True
+    )
+
+    process_lastfm_import.delay(lfm_import.id)
+
+    success_url = reverse_lazy('vrobbler-home')
+    return HttpResponseRedirect(success_url)
+
+
 @csrf_exempt
 @api_view(['GET'])
 def scrobble_endpoint(request):
@@ -98,10 +200,17 @@ def scrobble_endpoint(request):
 
 
 @csrf_exempt
+@permission_classes([IsAuthenticated])
 @api_view(['POST'])
 def jellyfin_websocket(request):
     data_dict = request.data
 
+    if (
+        data_dict['NotificationType'] == 'PlaybackProgress'
+        and data_dict['ItemType'] == 'Audio'
+    ):
+        return Response({}, status=status.HTTP_304_NOT_MODIFIED)
+
     # For making things easier to build new input processors
     if getattr(settings, "DUMP_REQUEST_DATA", False):
         json_data = json.dumps(data_dict, indent=4)
@@ -119,15 +228,18 @@ def jellyfin_websocket(request):
     if not scrobble:
         return Response({}, status=status.HTTP_400_BAD_REQUEST)
 
-    return Response(
-        {'scrobble_id': scrobble.id}, status=status.HTTP_201_CREATED
-    )
+    return Response({'scrobble_id': scrobble.id}, status=status.HTTP_200_OK)
 
 
 @csrf_exempt
+@permission_classes([IsAuthenticated])
 @api_view(['POST'])
 def mopidy_websocket(request):
-    data_dict = json.loads(request.data)
+    try:
+        data_dict = json.loads(request.data)
+    except TypeError:
+        logger.warning('Received Mopidy data as dict, rather than a string')
+        data_dict = request.data
 
     # For making things easier to build new input processors
     if getattr(settings, "DUMP_REQUEST_DATA", False):
@@ -142,6 +254,82 @@ def mopidy_websocket(request):
     if not scrobble:
         return Response({}, status=status.HTTP_400_BAD_REQUEST)
 
+    return Response({'scrobble_id': scrobble.id}, status=status.HTTP_200_OK)
+
+
+@csrf_exempt
+@permission_classes([IsAuthenticated])
+@api_view(['POST'])
+@parser_classes([MultiPartParser])
+def import_audioscrobbler_file(request):
+    """Takes a TSV file in the Audioscrobbler format, saves it and processes the
+    scrobbles.
+    """
+    scrobbles_created = []
+    # tsv_file = request.FILES[0]
+
+    file_serializer = AudioScrobblerTSVImportSerializer(data=request.data)
+    if file_serializer.is_valid():
+        import_file = file_serializer.save()
+        return Response(
+            {'scrobble_ids': scrobbles_created}, status=status.HTTP_200_OK
+        )
+    else:
+        return Response(
+            file_serializer.errors, status=status.HTTP_400_BAD_REQUEST
+        )
+
+
+@csrf_exempt
+@permission_classes([IsAuthenticated])
+@api_view(['GET'])
+def scrobble_finish(request, uuid):
+    user = request.user
+    if not user.is_authenticated:
+        return Response({}, status=status.HTTP_403_FORBIDDEN)
+
+    scrobble = Scrobble.objects.filter(user=user, uuid=uuid).first()
+    if not scrobble:
+        return Response({}, status=status.HTTP_404_NOT_FOUND)
+    scrobble.stop(force_finish=True)
     return Response(
-        {'scrobble_id': scrobble.id}, status=status.HTTP_201_CREATED
+        {'id': scrobble.id, 'status': scrobble.status},
+        status=status.HTTP_200_OK,
     )
+
+
+@csrf_exempt
+@permission_classes([IsAuthenticated])
+@api_view(['GET'])
+def scrobble_cancel(request, uuid):
+    user = request.user
+    if not user.is_authenticated:
+        return Response({}, status=status.HTTP_403_FORBIDDEN)
+
+    scrobble = Scrobble.objects.filter(user=user, uuid=uuid).first()
+    if not scrobble:
+        return Response({}, status=status.HTTP_404_NOT_FOUND)
+    scrobble.cancel()
+    return Response(
+        {'id': scrobble.id, 'status': 'cancelled'}, status=status.HTTP_200_OK
+    )
+
+
+@permission_classes([IsAuthenticated])
+@api_view(['GET'])
+def export(request):
+    format = request.GET.get('export_type', 'csv')
+    start = request.GET.get('start')
+    end = request.GET.get('end')
+    logger.debug(f"Exporting all scrobbles in format {format}")
+
+    temp_file, extension = export_scrobbles(
+        start_date=start, end_date=end, format=format
+    )
+
+    now = datetime.now()
+    filename = f"vrobbler-export-{str(now)}.{extension}"
+    response = FileResponse(open(temp_file, 'rb'))
+    response["Content-Disposition"] = f'attachment; filename="{filename}"'
+
+    return response

vrobbler/apps/sports/admin.py

@@ -0,0 +1,77 @@
+from django.contrib import admin
+
+from sports.models import (
+    League,
+    Player,
+    Round,
+    Season,
+    Sport,
+    SportEvent,
+    Team,
+)
+
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(Sport)
+class SportAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    ordering = ("name",)
+
+
+@admin.register(League)
+class LeagueAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name", "abbreviation_str")
+    ordering = ("name",)
+
+
+@admin.register(Player)
+class PlayerAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name", "league", "team")
+    ordering = ("name",)
+
+
+@admin.register(Season)
+class SeasonAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name", "league")
+    ordering = ("name",)
+
+
+@admin.register(Round)
+class RoundAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name", "season")
+    ordering = ("name",)
+
+
+@admin.register(Team)
+class TeamAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name", "league")
+    ordering = ("name",)
+
+
+@admin.register(SportEvent)
+class SportEventAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "title",
+        "event_type",
+        "start",
+        "comp_str",
+        "round",
+    )
+    list_filter = ("round__season", "home_team", "away_team")
+    ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]
+
+    def comp_str(self, obj):
+        if obj.home_team:
+            return f'{obj.away_team} @ {obj.home_team}'
+        if obj.player_one:
+            return f'{obj.player_one} v {obj.player_two}'

vrobbler/apps/sports/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class SportsConfig(AppConfig):
+    name = "sports"

vrobbler/apps/sports/migrations/0001_initial.py

@@ -0,0 +1,214 @@
+# Generated by Django 4.1.5 on 2023-01-14 21:14
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='League',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'logo',
+                    models.ImageField(
+                        blank=True, null=True, upload_to='sports/league-logos/'
+                    ),
+                ),
+                (
+                    'abbreviation_str',
+                    models.CharField(blank=True, max_length=10, null=True),
+                ),
+                ('thesportsdb_id', models.IntegerField(blank=True, null=True)),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Team',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ('thesportsdb_id', models.IntegerField(blank=True, null=True)),
+                (
+                    'league',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='sports.league',
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='SportEvent',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'title',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'run_time',
+                    models.CharField(blank=True, max_length=8, null=True),
+                ),
+                (
+                    'run_time_ticks',
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                (
+                    'event_type',
+                    models.CharField(
+                        choices=[
+                            ('UK', 'Unknown'),
+                            ('GA', 'Game'),
+                            ('MA', 'Match'),
+                            ('ME', 'Meet'),
+                        ],
+                        default='UK',
+                        max_length=2,
+                    ),
+                ),
+                ('start_utc', models.DateTimeField(blank=True, null=True)),
+                (
+                    'season',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'away_team',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        related_name='away_event_set',
+                        to='sports.team',
+                    ),
+                ),
+                (
+                    'home_team',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        related_name='home_event_set',
+                        to='sports.team',
+                    ),
+                ),
+                (
+                    'league',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='sports.league',
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/sports/migrations/0002_rename_start_utc_sportevent_start.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-01-14 21:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sports', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='sportevent',
+            old_name='start_utc',
+            new_name='start',
+        ),
+    ]

vrobbler/apps/sports/migrations/0003_sport_league_sport.py

@@ -0,0 +1,71 @@
+# Generated by Django 4.1.5 on 2023-01-21 04:21
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sports', '0002_rename_start_utc_sportevent_start'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Sport',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ('thesportsdb_id', models.IntegerField(blank=True, null=True)),
+                (
+                    'default_event_run_time',
+                    models.IntegerField(blank=True, null=True),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='league',
+            name='sport',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to='sports.sport',
+            ),
+        ),
+    ]

vrobbler/apps/sports/migrations/0004_alter_league_options_alter_sport_options_and_more.py

@@ -0,0 +1,254 @@
+# Generated by Django 4.1.5 on 2023-01-22 20:20
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sports', '0003_sport_league_sport'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='league',
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name='sport',
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name='team',
+            options={},
+        ),
+        migrations.RemoveField(
+            model_name='sportevent',
+            name='league',
+        ),
+        migrations.AlterField(
+            model_name='league',
+            name='thesportsdb_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='sport',
+            name='thesportsdb_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterField(
+            model_name='team',
+            name='thesportsdb_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.CreateModel(
+            name='Season',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'thesportsdb_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'league',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='sports.league',
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Round',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'thesportsdb_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'season',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='sports.season',
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Player',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'thesportsdb_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'league',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='sports.league',
+                    ),
+                ),
+                (
+                    'team',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='sports.team',
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='sportevent',
+            name='player_one',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                related_name='player_one_set',
+                to='sports.player',
+            ),
+        ),
+        migrations.AddField(
+            model_name='sportevent',
+            name='player_two',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                related_name='player_two_set',
+                to='sports.player',
+            ),
+        ),
+        migrations.AddField(
+            model_name='sportevent',
+            name='round',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to='sports.round',
+            ),
+        ),
+        migrations.AlterField(
+            model_name='sportevent',
+            name='season',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to='sports.season',
+            ),
+        ),
+    ]

vrobbler/apps/sports/migrations/0005_remove_sportevent_season.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.5 on 2023-01-22 20:34
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sports', '0004_alter_league_options_alter_sport_options_and_more'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='sportevent',
+            name='season',
+        ),
+    ]

vrobbler/apps/sports/migrations/0006_alter_sportevent_event_type.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.5 on 2023-01-22 21:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sports', '0005_remove_sportevent_season'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='sportevent',
+            name='event_type',
+            field=models.CharField(
+                choices=[('UK', 'Unknown'), ('GA', 'Game'), ('MA', 'Match')],
+                default='UK',
+                max_length=2,
+            ),
+        ),
+    ]

vrobbler/apps/sports/migrations/0007_sport_default_event_type.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.5 on 2023-01-22 22:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sports', '0006_alter_sportevent_event_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='sport',
+            name='default_event_type',
+            field=models.CharField(
+                choices=[('UK', 'Unknown'), ('GA', 'Game'), ('MA', 'Match')],
+                default='UK',
+                max_length=2,
+            ),
+        ),
+    ]

vrobbler/apps/sports/models.py

@@ -0,0 +1,224 @@
+import logging
+from typing import Dict
+from uuid import uuid4
+
+from django.conf import settings
+from django.db import models
+from django.urls import reverse
+from django.utils.translation import gettext_lazy as _
+from django_extensions.db.models import TimeStampedModel
+from scrobbles.mixins import ScrobblableMixin
+from sports.utils import get_players_from_event, get_round_name_from_event
+
+logger = logging.getLogger(__name__)
+BNULL = {"blank": True, "null": True}
+
+
+class SportEventType(models.TextChoices):
+    UNKNOWN = 'UK', _('Unknown')
+    GAME = 'GA', _('Game')
+    MATCH = 'MA', _('Match')
+
+
+class TheSportsDbMixin(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    thesportsdb_id = models.CharField(max_length=255, **BNULL)
+
+    class Meta:
+        abstract = True
+
+    def __str__(self):
+        return self.name
+
+
+class Sport(TheSportsDbMixin):
+    default_event_run_time = models.IntegerField(**BNULL)
+    default_event_type = models.CharField(
+        max_length=2,
+        choices=SportEventType.choices,
+        default=SportEventType.UNKNOWN,
+    )
+
+    # TODO Add these to the default run_time for Football
+    # run_time_seconds = 11700
+    # run_time_ticks = run_time_seconds * 1000
+    @property
+    def default_event_run_time_ticks(self):
+        default_run_time = getattr(
+            settings, 'DEFAULT_EVENT_RUNTIME_SECONDS', 14400
+        )
+        if self.default_event_run_time:
+            default_run_time = self.default_event_run_time
+        return default_run_time * 1000
+
+
+class League(TheSportsDbMixin):
+    logo = models.ImageField(upload_to="sports/league-logos/", **BNULL)
+    abbreviation_str = models.CharField(max_length=10, **BNULL)
+    sport = models.ForeignKey(Sport, on_delete=models.DO_NOTHING, **BNULL)
+
+    @property
+    def abbreviation(self):
+        return self.abbreviation_str
+
+
+class Season(TheSportsDbMixin):
+    league = models.ForeignKey(League, on_delete=models.DO_NOTHING, **BNULL)
+
+    def __str__(self):
+        return f'{self.name} season of {self.league}'
+
+
+class Team(TheSportsDbMixin):
+    league = models.ForeignKey(League, on_delete=models.DO_NOTHING, **BNULL)
+
+
+class Player(TheSportsDbMixin):
+    league = models.ForeignKey(League, on_delete=models.DO_NOTHING, **BNULL)
+    team = models.ForeignKey(Team, on_delete=models.DO_NOTHING, **BNULL)
+
+
+class Round(TheSportsDbMixin):
+    season = models.ForeignKey(Season, on_delete=models.DO_NOTHING, **BNULL)
+
+    def __str__(self):
+        return f'{self.name} of {self.season}'
+
+
+class SportEvent(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, 'SPORT_COMPLETION_PERCENT', 90)
+
+    event_type = models.CharField(
+        max_length=2,
+        choices=SportEventType.choices,
+        default=SportEventType.UNKNOWN,
+    )
+    round = models.ForeignKey(Round, on_delete=models.DO_NOTHING, **BNULL)
+    start = models.DateTimeField(**BNULL)
+    home_team = models.ForeignKey(
+        Team,
+        on_delete=models.DO_NOTHING,
+        related_name='home_event_set',
+        **BNULL,
+    )
+    away_team = models.ForeignKey(
+        Team,
+        on_delete=models.DO_NOTHING,
+        related_name='away_event_set',
+        **BNULL,
+    )
+    player_one = models.ForeignKey(
+        Player,
+        on_delete=models.DO_NOTHING,
+        related_name='player_one_set',
+        **BNULL,
+    )
+    player_two = models.ForeignKey(
+        Player,
+        on_delete=models.DO_NOTHING,
+        related_name='player_two_set',
+        **BNULL,
+    )
+
+    def __str__(self):
+        return f"{self.start.date()} - {self.round} - {self.home_team} v {self.away_team}"
+
+    def get_absolute_url(self):
+        return reverse("sports:event_detail", kwargs={'slug': self.uuid})
+
+    @classmethod
+    def find_or_create(cls, data_dict: Dict) -> "Event":
+        """Given a data dict from Jellyfin, does the heavy lifting of looking up
+        the video and, if need, TV Series, creating both if they don't yet
+        exist.
+
+        """
+        # Find or create our Sport
+        sid = data_dict.get("Sport")
+        sport, s_created = Sport.objects.get_or_create(thesportsdb_id=sid)
+        if s_created:
+            sport.name = sid
+            sport.save(update_fields=['name'])
+
+        # Find or create our League
+        lid = data_dict.get("LeagueId")
+        league, l_created = League.objects.get_or_create(
+            thesportsdb_id=lid, sport=sport
+        )
+        if l_created:
+            league.sport = sport
+            league.name = data_dict.get("LeagueName", "")
+            league.save(update_fields=['sport', 'name'])
+
+        # Find or create our Season
+        seid = data_dict.get('Season')
+        season, se_created = Season.objects.get_or_create(
+            thesportsdb_id=seid, league=league
+        )
+        if se_created:
+            season.name = seid
+            season.save(update_fields=['name'])
+
+        # Find or create our Round
+        rid = data_dict.get('RoundId')
+        round, r_created = Round.objects.get_or_create(
+            thesportsdb_id=rid, season=season
+        )
+        if r_created:
+            round.season = season
+            round.save(update_fields=['season'])
+
+        # Set some special data for Tennis
+        player_one = None
+        player_two = None
+        if data_dict.get('Sport') == 'Tennis':
+            event_name = data_dict.get('Name', '')
+            if not round.name:
+                round.name = get_round_name_from_event(event_name)
+                round.save(update_fields=['name'])
+
+            players_list = get_players_from_event(event_name)
+            player_one = Player.objects.filter(
+                name__icontains=players_list[0]
+            ).first()
+            if not player_one:
+                player_one = Player.objects.create(name=players_list[0])
+            player_two = Player.objects.filter(
+                name__icontains=players_list[1]
+            ).first()
+            if not player_two:
+                player_two = Player.objects.create(name=players_list[1])
+
+        home_team = None
+        away_team = None
+        if data_dict.get("HomeTeamName"):
+            home_team_dict = {
+                "name": data_dict.get("HomeTeamName", ""),
+                "thesportsdb_id": data_dict.get("HomeTeamId", ""),
+                "league": league,
+            }
+            home_team, _created = Team.objects.get_or_create(**home_team_dict)
+
+            away_team_dict = {
+                "name": data_dict.get("AwayTeamName", ""),
+                "thesportsdb_id": data_dict.get("AwayTeamId", ""),
+                "league": league,
+            }
+            away_team, _created = Team.objects.get_or_create(**away_team_dict)
+
+        event_dict = {
+            "title": data_dict.get("Name"),
+            "event_type": sport.default_event_type,
+            "home_team": home_team,
+            "away_team": away_team,
+            "player_one": player_one,
+            "player_two": player_two,
+            "start": data_dict['Start'],
+            "round": round,
+            "run_time_ticks": data_dict.get("RunTimeTicks"),
+            "run_time": data_dict.get("RunTime", ""),
+        }
+        event, _created = cls.objects.get_or_create(**event_dict)
+
+        return event

vrobbler/apps/sports/utils.py

@@ -0,0 +1,11 @@
+def get_round_name_from_event(event: str) -> str:
+    return ' '.join(event.split(' ')[:2])
+
+
+def get_players_from_event(event: str) -> list[str]:
+    players = []
+    event_name = get_round_name_from_event(event)
+    players_list = event.split(event_name)[1:][0].split('vs')
+    players.append(players_list[0].strip())
+    players.append(players_list[1].strip())
+    return players

vrobbler/apps/videos/admin.py

@@ -1,16 +1,20 @@
 from django.contrib import admin
-
+from scrobbles.models import Scrobble
 from videos.models import Series, Video
+from scrobbles.admin import ScrobbleInline
 
 
+@admin.register(Series)
 class SeriesAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     list_display = ("name", "tagline")
     ordering = ("-created",)
 
 
+@admin.register(Video)
 class VideoAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
+    raw_id_fields = ('tv_series',)
     list_display = (
         "title",
         "video_type",
@@ -22,7 +26,6 @@ class VideoAdmin(admin.ModelAdmin):
     )
     list_filter = ("year", "tv_series", "video_type")
     ordering = ("-created",)
-
-
-admin.site.register(Series, SeriesAdmin)
-admin.site.register(Video, VideoAdmin)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/videos/models.py

@@ -2,6 +2,7 @@ import logging
 from typing import Dict
 from uuid import uuid4
 
+from django.conf import settings
 from django.db import models
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
@@ -32,6 +33,9 @@ class Series(TimeStampedModel):
 
 
 class Video(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, 'VIDEO_COMPLETION_PERCENT', 90)
+    SECONDS_TO_STALE = getattr(settings, 'VIDEO_SECONDS_TO_STALE', 14400)
+
     class VideoType(models.TextChoices):
         UNKNOWN = 'U', _('Unknown')
         TV_EPISODE = 'E', _('TV Episode')
@@ -82,38 +86,38 @@ class Video(ScrobblableMixin):
             "video_type": Video.VideoType.MOVIE,
         }
 
+        series = None
         if data_dict.get("ItemType", "") == "Episode":
             series_name = data_dict.get("SeriesName", "")
             series, series_created = Series.objects.get_or_create(
                 name=series_name
             )
-            if series_created:
-                logger.debug(f"Created new series {series}")
-            else:
-                logger.debug(f"Found series {series}")
             video_dict['video_type'] = Video.VideoType.TV_EPISODE
 
         video, created = cls.objects.get_or_create(**video_dict)
 
+        run_time_ticks = data_dict.get("RunTimeTicks", None)
+        if run_time_ticks:
+            run_time_ticks = run_time_ticks // 10000
+
         video_extra_dict = {
             "year": data_dict.get("Year", ""),
             "overview": data_dict.get("Overview", None),
             "tagline": data_dict.get("Tagline", None),
-            "run_time_ticks": data_dict.get("RunTimeTicks", 0) // 10000,
+            "run_time_ticks": run_time_ticks,
             "run_time": convert_to_seconds(data_dict.get("RunTime", "")),
-            "tv_series_id": series.id,
             "tvdb_id": data_dict.get("Provider_tvdb", None),
             "tvrage_id": data_dict.get("Provider_tvrage", None),
-            "episode_number": data_dict.get("EpisodeNumber", ""),
-            "season_number": data_dict.get("SeasonNumber", ""),
+            "episode_number": data_dict.get("EpisodeNumber", None),
+            "season_number": data_dict.get("SeasonNumber", None),
         }
 
-        if created:
-            logger.debug(f"Created new video: {video}")
+        if series:
+            video_extra_dict["tv_series_id"] = series.id
+
+        if not video.run_time_ticks:
             for key, value in video_extra_dict.items():
                 setattr(video, key, value)
             video.save()
-        else:
-            logger.debug(f"Found video {video}")
 
         return video

vrobbler/apps/videos/urls.py

@@ -1,7 +1,7 @@
 from django.urls import path
 from videos import views
 
-app_name = 'scrobbles'
+app_name = 'videos'
 
 
 urlpatterns = [

vrobbler/celery.py

@@ -0,0 +1,13 @@
+import os
+
+from celery import Celery
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vrobbler.settings")
+app = Celery("vrobbler")
+app.config_from_object("django.conf:settings", namespace="CELERY")
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print(f"Request: {self.request!r}")

vrobbler/settings.py

@@ -37,10 +37,12 @@ KEEP_DETAILED_SCROBBLE_LOGS = os.getenv(
     "VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS", False
 )
 
-PODCAST_COMPLETION_PERCENT = os.getenv(
-    "VROBBLER_PODCAST_COMPLETION_PERCENT", 25
+# Key must be 16, 24 or 32 bytes long and will be converted to a byte stream
+ENCRYPTED_FIELD_KEY = os.getenv(
+    "VROBBLER_ENCRYPTED_FIELD_KEY", "12345678901234567890123456789012"
 )
-MUSIC_COMPLETION_PERCENT = os.getenv("VROBBLER_MUSIC_COMPLETION_PERCENT", 90)
+
+DJANGO_ENCRYPTED_FIELD_KEY = bytes(ENCRYPTED_FIELD_KEY, "utf-8")
 
 # Should we cull old in-progress scrobbles that are beyond the wait period for resuming?
 DELETE_STALE_SCROBBLES = os.getenv("VROBBLER_DELETE_STALE_SCROBBLES", True)
@@ -48,20 +50,20 @@ DELETE_STALE_SCROBBLES = os.getenv("VROBBLER_DELETE_STALE_SCROBBLES", True)
 # Used to dump data coming from srobbling sources, helpful for building new inputs
 DUMP_REQUEST_DATA = os.getenv("VROBBLER_DUMP_REQUEST_DATA", False)
 
-VIDEO_BACKOFF_MINUTES = os.getenv("VROBBLER_VIDEO_BACKOFF_MINUTES", 15)
-MUSIC_BACKOFF_SECONDS = os.getenv("VROBBLER_VIDEO_BACKOFF_SECONDS", 1)
-
-# If you stop waching or listening to a track, how long should we wait before we
-# give up on the old scrobble and start a new one? This could also be considered
-# a "continue in progress scrobble" time period. So if you pause the media and
-# start again, should it be a new scrobble.
-VIDEO_WAIT_PERIOD_DAYS = os.getenv("VROBBLER_VIDEO_WAIT_PERIOD_DAYS", 1)
-MUSIC_WAIT_PERIOD_MINUTES = os.getenv("VROBBLER_VIDEO_BACKOFF_MINUTES", 1)
 
+THESPORTSDB_API_KEY = os.getenv("VROBBLER_THESPORTSDB_API_KEY", "2")
+THESPORTSDB_BASE_URL = os.getenv(
+    "VROBBLER_THESPORTSDB_BASE_URL", "https://www.thesportsdb.com/api/v1/json/"
+)
 TMDB_API_KEY = os.getenv("VROBBLER_TMDB_API_KEY", "")
 
+LASTFM_API_KEY = os.getenv("VROBBLER_LASTFM_API_KEY")
+LASTFM_SECRET_KEY = os.getenv("VROBBLER_LASTFM_SECRET_KEY")
+
 DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
 
+TIME_ZONE = os.getenv("VROBBLER_TIME_ZONE", "US/Eastern")
+
 ALLOWED_HOSTS = ["*"]
 CSRF_TRUSTED_ORIGINS = [
     os.getenv("VROBBLER_TRUSTED_ORIGINS", "http://localhost:8000")
@@ -73,7 +75,7 @@ REDIS_URL = os.getenv("VROBBLER_REDIS_URL", None)
 CELERY_TASK_ALWAYS_EAGER = os.getenv("VROBBLER_SKIP_CELERY", False)
 CELERY_BROKER_URL = REDIS_URL if REDIS_URL else "memory://localhost/"
 CELERY_RESULT_BACKEND = "django-db"
-CELERY_TIMEZONE = os.getenv("VROBBLER_TIME_ZONE", "EST")
+CELERY_TIMEZONE = os.getenv("VROBBLER_TIME_ZONE", "US/Eastern")
 CELERY_TASK_TRACK_STARTED = True
 
 INSTALLED_APPS = [
@@ -87,11 +89,15 @@ INSTALLED_APPS = [
     "django.contrib.humanize",
     "django_filters",
     "django_extensions",
-    'rest_framework.authtoken',
+    "rest_framework.authtoken",
+    "encrypted_field",
+    "profiles",
     "scrobbles",
     "videos",
     "music",
     "podcasts",
+    "sports",
+    "mathfilters",
     "rest_framework",
     "allauth",
     "allauth.account",
@@ -154,9 +160,7 @@ CACHES = {
     }
 }
 if REDIS_URL:
-    CACHES["default"][
-        "BACKEND"
-    ] = "django.core.cache.backends.redis.RedisCache"
+    CACHES["default"]["BACKEND"] = "django_redis.cache.RedisCache"
     CACHES["default"]["LOCATION"] = REDIS_URL
 
 SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
@@ -169,8 +173,8 @@ AUTHENTICATION_BACKENDS = [
 REST_FRAMEWORK = {
     "DEFAULT_PERMISSION_CLASSES": ("rest_framework.permissions.AllowAny",),
     'DEFAULT_AUTHENTICATION_CLASSES': [
-        #'rest_framework.authentication.BasicAuthentication',
-        #'rest_framework.authentication.TokenAuthentication',
+        'rest_framework.authentication.BasicAuthentication',
+        'rest_framework.authentication.TokenAuthentication',
         'rest_framework.authentication.SessionAuthentication',
     ],
     "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
@@ -211,8 +215,6 @@ TIME_ZONE = os.getenv("VROBBLER_TIME_ZONE", "EST")
 
 USE_I18N = True
 
-USE_L10N = True
-
 USE_TZ = True
 
 
@@ -223,10 +225,6 @@ STATIC_URL = "static/"
 STATIC_ROOT = os.getenv(
     "VROBBLER_STATIC_ROOT", os.path.join(PROJECT_ROOT, "static")
 )
-if not DEBUG:
-    STATICFILES_STORAGE = (
-        "whitenoise.storage.CompressedManifestStaticFilesStorage"
-    )
 
 MEDIA_URL = "/media/"
 MEDIA_ROOT = os.getenv(
@@ -281,17 +279,17 @@ LOGGING = {
             "class": "logging.NullHandler",
             "level": LOG_LEVEL,
         },
-        "file": {
-            "class": "logging.handlers.RotatingFileHandler",
-            "filename": "".join([LOG_FILE_PATH, "vrobbler.log"]),
-            "formatter": LOG_TYPE,
-            "level": LOG_LEVEL,
+        'sql': {
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': ''.join([LOG_FILE_PATH, 'vrobbler_sql.', LOG_TYPE]),
+            'formatter': LOG_TYPE,
+            'level': LOG_LEVEL,
         },
-        "requests_file": {
-            "class": "logging.handlers.RotatingFileHandler",
-            "filename": "".join([LOG_FILE_PATH, "vrobbler_requests.log"]),
-            "formatter": LOG_TYPE,
-            "level": LOG_LEVEL,
+        'file': {
+            'class': 'logging.handlers.RotatingFileHandler',
+            'filename': ''.join([LOG_FILE_PATH, 'vrobbler.', LOG_TYPE]),
+            'formatter': LOG_TYPE,
+            'level': LOG_LEVEL,
         },
     },
     "loggers": {
@@ -301,13 +299,18 @@ LOGGING = {
             "propagate": True,
         },
         "django.db.backends": {"handlers": ["null"]},
+        "django.server": {"handlers": ["null"]},
+        "pylast": {"handlers": ["null"], "propagate": False},
+        "musicbrainzngs": {"handlers": ["null"], "propagate": False},
+        "httpx": {"handlers": ["null"], "propagate": False},
         "vrobbler": {
-            "handlers": ["console", "file"],
-            "propagate": True,
+            "handlers": ["console"],
+            "propagate": False,
         },
     },
 }
 
-if DEBUG:
-    # We clear out a db with lots of games all the time in dev
-    DATA_UPLOAD_MAX_NUMBER_FIELDS = 3000
+LOG_TO_CONSOLE = os.getenv("VROBBLER_LOG_TO_CONSOLE", False)
+if LOG_TO_CONSOLE:
+    LOGGING['loggers']['django']['handlers'] = ["console"]
+    LOGGING['loggers']['vrobbler']['handlers'] = ["console"]

vrobbler/templates/base.html

@@ -8,7 +8,9 @@
         <meta http-equiv="x-ua-compatible" content="ie=edge">
         <meta name="description" content="">
         <meta name="viewport" content="width=device-width, initial-scale=1">
+        <link rel="shortcut icon" type="image/png" href="{% static 'images/favicon.ico' %}"/>
         <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-EVSTQN3/azprG1Anm3QDgpJLIm9Nao0Yz1ztcQTwFspd3yD65VohhpuuCOmLASjC" crossorigin="anonymous">
+        <script src="https://cdn.jsdelivr.net/npm/popper.js@1.12.9/dist/umd/popper.min.js" crossorigin="anonymous"></script>
         <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.0.2/dist/js/bootstrap.bundle.min.js" integrity="sha384-MrcW6ZMFYlzcLA8Nl+NtUVF0sA7MsXsP1UyJoMp4YLEuNSfAP+JcXn/tWtIaxVXM" crossorigin="anonymous"></script>
         <script src="https://code.jquery.com/jquery-3.3.1.js" integrity="sha256-2Kok7MbOyxpgUVvAk/HJ2jigOSYS2auK4Pfzbm7uH60=" crossorigin="anonymous"></script>
         <script src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.10/clipboard.min.js"></script>
@@ -161,12 +163,11 @@
         border-color: transparent;
         box-shadow: 0 0 0 3px rgba(255, 255, 255, .25);
         }
-
+        #scrobble-form { width: 100% }
         </style>
         {% block head_extra %}{% endblock %}
 
-        <link rel="apple-touch-icon" href="/apple-touch-icon.png">
-        <!-- Place favicon.ico in the root directory -->
+        <link rel="apple-touch-icon" href="{% static 'images/apple-touch-icon.png' %}">
     </head>
     <body>
         <header class="navbar navbar-dark sticky-top bg-dark flex-md-nowrap p-0 shadow">
@@ -174,13 +175,19 @@
             <button class="navbar-toggler position-absolute d-md-none collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#sidebarMenu" aria-controls="sidebarMenu" aria-expanded="false" aria-label="Toggle navigation">
                 <span class="navbar-toggler-icon"></span>
             </button>
-            <input class="form-control form-control-dark w-100" type="text" placeholder="Search" aria-label="Search">
+
+            {% if user.is_authenticated %}
+            <form id="scrobble-form" action="{% url 'imdb-manual-scrobble' %}" method="post">
+                {% csrf_token %}
+                {{ imdb_form }}
+            </form>
+            {% endif %}
             <div class="navbar-nav">
                 <div class="nav-item text-nowrap">
-                {% if user.is_authenticated %}
-                <a class="nav-link px-3" href="{% url "account_logout" %}">Sign out</a>
-                {% else %}
-                <a class="nav-link px-3" href="{% url "account_login" %}">Sign in</a>
+                    {% if user.is_authenticated %}
+                    <a class="nav-link px-3" href="{% url "account_logout" %}">Sign out</a>
+                    {% else %}
+                    <a class="nav-link px-3" href="{% url "account_login" %}">Sign in</a>
                 {% endif %}
                 </div>
             </div>
@@ -190,7 +197,7 @@
             <div class="row">
                 <nav id="sidebarMenu" class="col-md-3 col-lg-2 d-md-block bg-light sidebar collapse">
                     <div class="position-sticky pt-3">
-                        {% if now_playing_list %}
+                        {% if now_playing_list and user.is_authenticated %}
                         <ul style="padding-right:10px;">
                             <b>Now playing</b>
                             {% for scrobble in now_playing_list %}
@@ -199,11 +206,14 @@
                                 {% if scrobble.track %}<em>{{scrobble.track.artist}}</em><br/>{% endif %}
                                 {% if scrobble.podcast_episode%}<em>{{scrobble.podcast_episode.podcast}}</em><br/>{% endif %}
                                 {% if scrobble.video.tv_series %}<em>{{scrobble.video.tv_series }}</em><br/>{% endif %}
-                                <small>{{scrobble.created|naturaltime}}<br/>
+                                {% if scrobble.sport_event %}<em>{{scrobble.sport_event.round.season.league}}</em><br/>{% endif %}
+                                <small>{{scrobble.timestamp|naturaltime}}<br/>
                                     from {{scrobble.source}}</small>
                                 <div class="progress-bar" style="margin-right:5px;">
                                     <span class="progress-bar-fill" style="width: {{scrobble.percent_played}}%;"></span>
                                 </div>
+                                <a href="{% url "scrobbles:cancel" scrobble.uuid %}">Cancel</a>
+                                <a href="{% url "scrobbles:finish" scrobble.uuid %}">Finish</a>
                             </div>
                             <hr/>
                             {% endfor %}
@@ -213,7 +223,7 @@
 
                         <ul class="nav flex-column">
                             <li class="nav-item">
-                                <a class="nav-link active" aria-current="page" href="#">
+                                <a class="nav-link active" aria-current="page" href="/">
                                 <span data-feather="music"></span>
                                 Dashboard
                                 </a>
@@ -261,7 +271,6 @@
                 {% endblock %}
             </div>
         </div>
-        {% block extra_js %}
         <script src="https://cdn.jsdelivr.net/npm/feather-icons@4.28.0/dist/feather.min.js" integrity="sha384-uO3SXW5IuS1ZpFPKugNNWqTZRRglnUJK6UAZ/gxOX80nxEkN9NcGZTftn6RzhGWE" crossorigin="anonymous"></script><script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.min.js" integrity="sha384-zNy6FEbO50N+Cg5wap8IKA4M/ZnLJgzc6w2NqACZaK0u0FXfOWRRJOnQtpZun8ha" crossorigin="anonymous"></script>
         <script><!-- comment ------------------------------------------------->
             /* globals Chart:false, feather:false */
@@ -310,6 +319,7 @@
             })()
 
         </script>
+        {% block extra_js %}
         {% endblock %}
     </body>
 </html>

vrobbler/templates/base_detail.html

@@ -0,0 +1,18 @@
+{% extends "base.html" %}
+
+{% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">{% block title %}{% endblock %} </h1>
+        <div class="btn-toolbar mb-2 mb-md-0">
+            <div class="btn-group me-2">
+                <button type="button" class="btn btn-sm btn-outline-secondary">Export</button>
+            </div>
+        </div>
+    </div>
+
+    <div class="container">
+        {% block details %}{% endblock %}
+    </div>
+</main>
+{% endblock %}

vrobbler/templates/base_list.html

@@ -0,0 +1,18 @@
+{% extends "base.html" %}
+
+{% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">{% block title %}{% endblock %} </h1>
+        <div class="btn-toolbar mb-2 mb-md-0">
+            <div class="btn-group me-2">
+                <button type="button" class="btn btn-sm btn-outline-secondary">Export</button>
+            </div>
+        </div>
+    </div>
+
+    <div class="container">
+        {% block lists %}{% endblock %}
+    </div>
+</main>
+{% endblock %}

vrobbler/templates/music/album_list.html

@@ -0,0 +1,11 @@
+{% extends "base_list.html" %}
+
+{% block title %}Albums{% endblock %}
+
+{% block lists %}
+{% for album in object_list %}
+<dl style="width: 130px; float: left; margin-right:10px;">
+    <dd><img src="{{album.cover_image.url}}" width=120 height=120 /></dd>
+</dl>
+{% endfor %}
+{% endblock %}

vrobbler/templates/music/artist_detail.html

@@ -0,0 +1,72 @@
+{% extends "base_detail.html" %}
+{% load mathfilters %}
+
+{% block title %}{{object.name}}{% endblock %}
+
+{% block details %}
+
+<div class="row">
+    {% for album in artist.album_set.all %}
+    {% if album.cover_image %}
+    <p style="width:150px; float:left;"><img src="{{album.cover_image.url}}" width=150 height=150 /></p>
+    {% endif %}
+    {% endfor %}
+</div>
+<div class="row">
+    <p>{{artist.scrobbles.count}} scrobbles</p>
+    <div class="col-md">
+        <h3>Top tracks</h3>
+        <div class="table-responsive">
+            <table class="table table-striped table-sm">
+            <thead>
+                <tr>
+                    <th scope="col">Rank</th>
+                    <th scope="col">Track</th>
+                    <th scope="col">Count</th>
+                    <th scope="col"></th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for track in object.tracks %}
+                <tr>
+                    <td>{{rank}}#1</td>
+                    <td>{{track.title}}</td>
+                    <td>{{track.scrobble_count}}</td>
+                    <td>
+                        <div class="progress-bar" style="margin-right:5px;">
+                            <span class="progress-bar-fill" style="width: {{track.scrobble_count|mul:10}}%;"></span>
+                        </div>
+                    </td>
+                </tr>
+                {% endfor %}
+            </tbody>
+            </table>
+        </div>
+    </div>
+</div>
+<div class="row">
+    <div class="col-md">
+        <h3>Last scrobbles</h3>
+        <div class="table-responsive">
+            <table class="table table-striped table-sm">
+            <thead>
+                <tr>
+                    <th scope="col">Date</th>
+                    <th scope="col">Track</th>
+                    <th scope="col">Album</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for scrobble in object.scrobbles %}
+                <tr>
+                    <td>{{scrobble.timestamp}}</td>
+                    <td>{{scrobble.track.title}}</td>
+                    <td>{{scrobble.track.album.name}}</td>
+                </tr>
+                {% endfor %}
+            </tbody>
+            </table>
+        </div>
+    </div>
+</div>
+{% endblock %}

vrobbler/templates/music/artist_list.html

@@ -0,0 +1,30 @@
+{% extends "base_list.html" %}
+
+{% block title %}Artists{% endblock %}
+
+{% block lists %}
+<div class="row">
+    <div class="col-md">
+        <div class="table-responsive">
+            <table class="table table-striped table-sm">
+            <thead>
+                <tr>
+                    <th scope="col">Artist</th>
+                    <th scope="col">Scrobbles</th>
+                    <th scope="col">All time</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for artist in object_list %}
+                <tr>
+                    <td><a href="{{artist.get_absolute_url}}">{{artist}}</a></td>
+                    <td>{{artist.scrobbles.count}}</td>
+                    <td></td>
+                </tr>
+                {% endfor %}
+            </tbody>
+            </table>
+        </div>
+    </div>
+</div>
+{% endblock %}

vrobbler/templates/music/track_detail.html

@@ -0,0 +1,13 @@
+{% extends "base_detail.html" %}
+
+{% block title %}{{object.title}}{% endblock %}
+
+{% block details %}
+<h2>Last scrobbles</h2>
+{% for scrobble in object.scrobble_set.all  %}
+<ul>
+    <li>{{scrobble.timestamp|date:"d M Y h:m"}} - <img src="{{object.album.cover_image.url}}" width=25 height=25 /> - {{object}}</li>
+</ul>
+{% endfor %}
+
+{% endblock %}

vrobbler/templates/music/track_list.html

@@ -0,0 +1,12 @@
+{% extends "base_list.html" %}
+
+{% block title %}Tracks{% endblock %}
+
+{% block lists %}
+<h2>All time</h2>
+{% for track in object_list %}
+<ul>
+    <li><a href="{{track.get_absolute_url}}">{{track}}</a></li>
+</ul>
+{% endfor %}
+{% endblock %}

vrobbler/templates/scrobbles/manual_form.html

@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+{% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">Manual scrobble</h1>
+        <form action="#" method="post">
+            {% csrf_token %}
+            {{ form }}
+            <input type="submit" value="Submit">
+        </form>
+    </div>
+</main>
+{% endblock %}

vrobbler/templates/scrobbles/scrobble_list.html

@@ -4,17 +4,35 @@
 
 {% block content %}
 <main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
-    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+    <div
+        class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
         <h1 class="h2">Dashboard</h1>
         <div class="btn-toolbar mb-2 mb-md-0">
-        <div class="btn-group me-2">
-            <button type="button" class="btn btn-sm btn-outline-secondary">Share</button>
-            <button type="button" class="btn btn-sm btn-outline-secondary">Export</button>
-        </div>
-        <button type="button" class="btn btn-sm btn-outline-secondary dropdown-toggle">
-            <span data-feather="calendar"></span>
-            This week
-        </button>
+            {% if user.is_authenticated %}
+            <div class="btn-group me-2">
+                {% if user.profile.lastfm_username %}
+
+                <form action="{% url 'scrobbles:lastfm-import' %}" method="get">
+                    <button type="submit" class="btn btn-sm btn-outline-secondary">Last.fm Sync</button>
+                </form>
+                {% endif %}
+                <button type="button" class="btn btn-sm btn-outline-secondary" data-bs-toggle="modal"
+                    data-bs-target="#importModal">Import</button>
+                <button type="button" class="btn btn-sm btn-outline-secondary" data-bs-toggle="modal"
+                    data-bs-target="#exportModal">Export</button>
+            </div>
+            {% endif %}
+            <div class="dropdown">
+                <button type="button" class="btn btn-sm btn-outline-secondary dropdown-toggle" id="graphDateButton"
+                    data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
+                    <span data-feather="calendar"></span>
+                    This week
+                </button>
+                <div class="dropdown-menu" data-bs-toggle="#graphDataChange" aria-labelledby="graphDateButton">
+                    <a class="dropdown-item" href="#">This month</a>
+                    <a class="dropdown-item" href="#">This year</a>
+                </div>
+            </div>
         </div>
     </div>
 
@@ -22,130 +40,329 @@
 
     <div class="container">
 
+        {% if user.is_authenticated %}
         <div class="row">
-            <p>Today <b>{{counts.today}}</b> | This Week <b>{{counts.week}}</b> | This Month <b>{{counts.month}}</b> | This Year <b>{{counts.year}}</b> | All Time <b>{{counts.alltime}}</b></p>
+            <p>Today <b>{{counts.today}}</b> | This Week <b>{{counts.week}}</b> | This Month <b>{{counts.month}}</b> |
+                This Year <b>{{counts.year}}</b> | All Time <b>{{counts.alltime}}</b></p>
         </div>
         <div class="row">
             <div class="col-md">
                 <ul class="nav nav-tabs" id="myTab" role="tablist">
                     <li class="nav-item" role="presentation">
-                        <button class="nav-link active" id="home-tab" data-bs-toggle="tab" data-bs-target="#artists-week" type="button" role="tab" aria-controls="home" aria-selected="true">Weekly Artists</button>
+                        <button class="nav-link active" id="home-tab" data-bs-toggle="tab"
+                            data-bs-target="#artists-week" type="button" role="tab" aria-controls="home"
+                            aria-selected="true">Weekly Artists</button>
                     </li>
                     <li class="nav-item" role="presentation">
-                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#tracks-week" type="button" role="tab" aria-controls="profile" aria-selected="false">Weekly Tracks</button>
+                        <button class="nav-link" id="artist-month-tab" data-bs-toggle="tab"
+                            data-bs-target="#artists-month" type="button" role="tab" aria-controls="home"
+                            aria-selected="true">Monthly Artists</button>
+                    </li>
+                    <li class="nav-item" role="presentation">
+                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#tracks-week"
+                            type="button" role="tab" aria-controls="profile" aria-selected="false">Weekly
+                            Tracks</button>
+                    </li>
+                    <li class="nav-item" role="presentation">
+                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#tracks-month"
+                            type="button" role="tab" aria-controls="profile" aria-selected="false">Monthly
+                            Tracks</button>
                     </li>
                 </ul>
 
                 <div class="tab-content" id="myTabContent">
-                    <div class="tab-pane fade show" id="artists-week" role="tabpanel" aria-labelledby="artists-week-tab">
+                    <div class="tab-pane fade show active" id="artists-week" role="tabpanel"
+                        aria-labelledby="artists-week-tab">
                         <h2>Top artists this week</h2>
                         <div class="table-responsive">
                             <table class="table table-striped table-sm">
-                            <thead>
-                                <tr>
-                                    <th scope="col">#</th>
-                                    <th scope="col">Artist</th>
-                                </tr>
-                            </thead>
-                            <tbody>
-                                {% for artist in top_weekly_artists %}
-                                <tr>
-                                    <td>{{artist.num_scrobbles}}</td>
-                                    <td>{{artist.name}}</td>
-                                </tr>
-                                {% endfor %}
-                            </tbody>
+                                <thead>
+                                    <tr>
+                                        <th scope="col">#</th>
+                                        <th scope="col">Artist</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for artist in top_weekly_artists %}
+                                    <tr>
+                                        <td>{{artist.num_scrobbles}}</td>
+                                        <td>{{artist.name}}</td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
                             </table>
                         </div>
                     </div>
 
-                    <div class="tab-pane fade show active" id="tracks-week" role="tabpanel" aria-labelledby="tracks-week-tab">
+                    <div class="tab-pane fade show" id="tracks-week" role="tabpanel" aria-labelledby="tracks-week-tab">
                         <h2>Top tracks this week</h2>
                         <div class="table-responsive">
                             <table class="table table-striped table-sm">
-                            <thead>
-                                <tr>
-                                    <th scope="col">#</th>
-                                    <th scope="col">Track</th>
-                                    <th scope="col">Artist</th>
-                                </tr>
-                            </thead>
-                            <tbody>
-                                {% for track in top_weekly_tracks %}
-                                <tr>
-                                    <td>{{track.num_scrobbles}}</td>
-                                    <td>{{track.title}}</td>
-                                    <td>{{track.artist.name}}</td>
-                                </tr>
-                                {% endfor %}
-                            </tbody>
+                                <thead>
+                                    <tr>
+                                        <th scope="col">#</th>
+                                        <th scope="col">Track</th>
+                                        <th scope="col">Artist</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for track in top_weekly_tracks %}
+                                    <tr>
+                                        <td>{{track.num_scrobbles}}</td>
+                                        <td>{{track.title}}</td>
+                                        <td>{{track.artist.name}}</td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
                             </table>
                         </div>
                     </div>
+
+
+                    <div class="tab-pane fade show" id="tracks-month" role="tabpanel"
+                        aria-labelledby="tracks-month-tab">
+                        <h2>Top tracks this month</h2>
+                        <div class="table-responsive">
+                            <table class="table table-striped table-sm">
+                                <thead>
+                                    <tr>
+                                        <th scope="col">#</th>
+                                        <th scope="col">Track</th>
+                                        <th scope="col">Artist</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for track in top_monthly_tracks %}
+                                    <tr>
+                                        <td>{{track.num_scrobbles}}</td>
+                                        <td>{{track.title}}</td>
+                                        <td>{{track.artist.name}}</td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
+                            </table>
+                        </div>
+                    </div>
+
+                    <div class="tab-pane fade show " id="artists-month" role="tabpanel"
+                        aria-labelledby="artists-month-tab">
+                        <h2>Top artists this month</h2>
+                        <div class="table-responsive">
+                            <table class="table table-striped table-sm">
+                                <thead>
+                                    <tr>
+                                        <th scope="col">#</th>
+                                        <th scope="col">Artist</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for artist in top_monthly_artists %}
+                                    <tr>
+                                        <td>{{artist.num_scrobbles}}</td>
+                                        <td>{{artist.name}}</td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
+                            </table>
+                        </div>
+                    </div>
+
                 </div>
             </div>
-        <div class="col-md">
-            <ul class="nav nav-tabs" id="myTab" role="tablist">
-                <li class="nav-item" role="presentation">
-                    <button class="nav-link active" id="home-tab" data-bs-toggle="tab" data-bs-target="#latest-listened" type="button" role="tab" aria-controls="home" aria-selected="true">Latest Listened</button>
-                </li>
-                <li class="nav-item" role="presentation">
-                    <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-watched" type="button" role="tab" aria-controls="profile" aria-selected="false">Latest Watched</button>
-                </li>
-            </ul>
+            <div class="col-md">
+                <ul class="nav nav-tabs" id="myTab" role="tablist">
+                    <li class="nav-item" role="presentation">
+                        <button class="nav-link active" id="home-tab" data-bs-toggle="tab"
+                            data-bs-target="#latest-listened" type="button" role="tab" aria-controls="home"
+                            aria-selected="true">Tracks</button>
+                    </li>
+                    <li class="nav-item" role="presentation">
+                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-watched"
+                            type="button" role="tab" aria-controls="profile" aria-selected="false">Videos</button>
+                    </li>
+                    <li class="nav-item" role="presentation">
+                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab"
+                            data-bs-target="#latest-podcasted" type="button" role="tab" aria-controls="profile"
+                            aria-selected="false">Podcasts</button>
+                    </li>
+                    <li class="nav-item" role="presentation">
+                        <button class="nav-link" id="profile-tab" data-bs-toggle="tab" data-bs-target="#latest-sports"
+                            type="button" role="tab" aria-controls="profile" aria-selected="false">Sports</button>
+                    </li>
+                </ul>
 
-            <div class="tab-content" id="myTabContent2">
-                <div class="tab-pane fade show active" id="latest-listened" role="tabpanel" aria-labelledby="latest-listened-tab">
-                    <h2>Latest listened</h2>
-                    <div class="table-responsive">
-                        <table class="table table-striped table-sm">
-                        <thead>
-                            <tr>
-                            <th scope="col">Time</th>
-                            <th scope="col">Track</th>
-                            <th scope="col">Artist</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                            {% for scrobble in object_list %}
-                            <tr>
-                                <td>{{scrobble.timestamp|naturaltime}}</td>
-                                <td>{{scrobble.track.title}}</td>
-                                <td>{{scrobble.track.artist.name}}</td>
-                            </tr>
-                            {% endfor %}
-                        </tbody>
-                        </table>
+                <div class="tab-content" id="myTabContent2">
+                    <div class="tab-pane fade show active" id="latest-listened" role="tabpanel"
+                        aria-labelledby="latest-listened-tab">
+                        <h2>Latest listened</h2>
+                        <div class="table-responsive">
+                            <table class="table table-striped table-sm">
+                                <thead>
+                                    <tr>
+                                        <th scope="col">Time</th>
+                                        <th scope="col">Album</th>
+                                        <th scope="col">Track</th>
+                                        <th scope="col">Artist</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for scrobble in object_list %}
+                                    <tr>
+                                        <td>{{scrobble.timestamp|naturaltime}}</td>
+                                        {% if scrobble.track.album.cover_image %}
+                                        <td><img src="{{scrobble.track.album.cover_image.url}}" width=50 height=50
+                                                style="border:1px solid black;" /></td>
+                                        {% else %}
+                                        <td>{{scrobble.track.album.name}}</td>
+                                        {% endif %}
+                                        <td>{{scrobble.track.title}}</td>
+                                        <td>{{scrobble.track.artist.name}}</td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
+                            </table>
+                        </div>
                     </div>
-                </div>
 
-                <div class="tab-pane fade show" id="latest-watched" role="tabpanel" aria-labelledby="latest-watched-tab">
-                    <h2>Latest watched</h2>
-                    <div class="table-responsive">
-                        <table class="table table-striped table-sm">
-                        <thead>
-                            <tr>
-                            <th scope="col">Time</th>
-                            <th scope="col">Title</th>
-                            <th scope="col">Series</th>
-                            <th scope="col">Source</th>
-                            </tr>
-                        </thead>
-                        <tbody>
-                            {% for scrobble in video_scrobble_list %}
-                            <tr>
-                                <td>{{scrobble.timestamp|naturaltime}}</td>
-                                <td>{% if scrobble.video.tv_series %}E{{scrobble.video.season_number}}S{{scrobble.video.season_number}} -{% endif %} {{scrobble.video.title}}</td>
-                                <td>{% if scrobble.video.tv_series %}{{scrobble.video.tv_series}}{% endif %}</td>
-                                <td>{{scrobble.source}}</td>
-                            </tr>
-                            {% endfor %}
-                        </tbody>
-                        </table>
+                    <div class="tab-pane fade show" id="latest-watched" role="tabpanel"
+                        aria-labelledby="latest-watched-tab">
+                        <h2>Latest watched</h2>
+                        <div class="table-responsive">
+                            <table class="table table-striped table-sm">
+                                <thead>
+                                    <tr>
+                                        <th scope="col">Time</th>
+                                        <th scope="col">Title</th>
+                                        <th scope="col">Series</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for scrobble in video_scrobble_list %}
+                                    <tr>
+                                        <td>{{scrobble.timestamp|naturaltime}}</td>
+                                        <td>{% if scrobble.video.tv_series %}S{{scrobble.video.season_number}}E{{scrobble.video.episode_number}} -{% endif %} {{scrobble.video.title}}</td>
+                                        <td>{% if scrobble.video.tv_series %}{{scrobble.video.tv_series}}{% endif %}
+                                        </td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
+                            </table>
+                        </div>
                     </div>
+
+                    <div class="tab-pane fade show" id="latest-sports" role="tabpanel"
+                        aria-labelledby="latest-sports-tab">
+                        <h2>Latest Sports</h2>
+                        <div class="table-responsive">
+                            <table class="table table-striped table-sm">
+                                <thead>
+                                    <tr>
+                                        <th scope="col">Date</th>
+                                        <th scope="col">Title</th>
+                                        <th scope="col">League</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for scrobble in sport_scrobble_list %}
+                                    <tr>
+                                        <td>{{scrobble.timestamp|naturaltime}}</td>
+                                        <td>{{scrobble.sport_event.title}}</td>
+                                        <td>{{scrobble.sport_event.league.abbreviation}}</td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
+                            </table>
+                        </div>
+                    </div>
+
+                    <div class="tab-pane fade show" id="latest-podcasted" role="tabpanel"
+                        aria-labelledby="latest-podcasted-tab">
+                        <h2>Latest Podcasted</h2>
+                        <div class="table-responsive">
+                            <table class="table table-striped table-sm">
+                                <thead>
+                                    <tr>
+                                        <th scope="col">Date</th>
+                                        <th scope="col">Title</th>
+                                        <th scope="col">Podcast</th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    {% for scrobble in podcast_scrobble_list %}
+                                    <tr>
+                                        <td>{{scrobble.timestamp|naturaltime}}</td>
+                                        <td>{{scrobble.podcast_episode.title}}</td>
+                                        <td>{{scrobble.podcast_episode.podcast}}</td>
+                                    </tr>
+                                    {% endfor %}
+                                </tbody>
+                            </table>
+                        </div>
+                    </div>
+
                 </div>
             </div>
+            {% endif %}
+        </div>
+</main>
+
+<div class="modal fade" id="importModal" tabindex="-1" role="dialog" aria-labelledby="importModalLabel"
+    aria-hidden="true">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title" id="importModalLabel">Import scrobbles</h5>
+                <button type="button" class="close" data-bs-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+            </div>
+            <form action="{% url 'audioscrobbler-file-upload' %}" method="post" enctype="multipart/form-data">
+                <div class="modal-body">
+                    {% csrf_token %}
+                    <div class="form-group">
+                        <label for="tsv_file" class="col-form-label">Audioscrobbler TSV file:</label>
+                        <input type="file" name="tsv_file" class="form-control" id="id_tsv_file">
+                    </div>
+                </div>
+                <div class="modal-footer">
+                    <button type="submit" class="btn btn-primary">Import</button>
+                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+                </div>
+            </form>
         </div>
     </div>
-</main>
+</div>
+
+<div class="modal fade" id="exportModal" tabindex="-1" role="dialog" aria-labelledby="exportModalLabel"
+    aria-hidden="true">
+    <div class="modal-dialog" role="document">
+        <div class="modal-content">
+            <div class="modal-header">
+                <h5 class="modal-title" id="exportModalLabel">Export scrobbles</h5>
+                <button type="button" class="close" data-bs-dismiss="modal" aria-label="Close">
+                    <span aria-hidden="true">&times;</span>
+                </button>
+            </div>
+            <form action="{% url 'scrobbles:export' %}" method="get">
+                <div class="modal-body">
+                    {% csrf_token %}
+                    <div class="form-group">
+                        {{export_form.as_div}}
+                    </div>
+                </div>
+                <div class="modal-footer">
+                    <button type="submit" class="btn btn-primary">Export</button>
+                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+                </div>
+            </form>
+        </div>
+    </div>
+</div>
+{% endblock %}
+
+{% block extra_js %}
+<script>
+    $('#importModal').on('shown.bs.modal', function () { $('#importInput').trigger('focus') });
+    $('#exportModal').on('shown.bs.modal', function () { $('#exportInput').trigger('focus') });
+</script>
 {% endblock %}

vrobbler/templates/scrobbles/upload_form.html

@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+{% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">Manual scrobble</h1>
+        <form action="{% url 'audioscrobbler-file-upload' %}" method="post">
+            {% csrf_token %}
+            {{ form }}
+            <input type="submit" value="Submit">
+        </form>
+    </div>
+</main>
+{% endblock %}

vrobbler/templates/videos/movie_list.html

@@ -1,12 +1,28 @@
 {% extends "base.html" %}
 
 {% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">Movies</h1>
+        <div class="btn-toolbar mb-2 mb-md-0">
+        <div class="btn-group me-2">
+            <button type="button" class="btn btn-sm btn-outline-secondary">Share</button>
+            <button type="button" class="btn btn-sm btn-outline-secondary">Export</button>
+        </div>
+        <button type="button" class="btn btn-sm btn-outline-secondary dropdown-toggle">
+            <span data-feather="calendar"></span>
+            This week
+        </button>
+        </div>
+    </div>
 
-    <h2>Movies</h2>
+    <div class="container">
 
     <ul>
         {% for movie in object_list %}
         <li>{{movie}}</li>
         {% endfor %}
     </ul>
+    </div>
+</main>
 {% endblock %}

vrobbler/templates/videos/series_list.html

@@ -0,0 +1,22 @@
+{% extends "base.html" %}
+
+{% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">Series</h1>
+        <div class="btn-toolbar mb-2 mb-md-0">
+            <div class="btn-group me-2">
+                <button type="button" class="btn btn-sm btn-outline-secondary">Export</button>
+            </div>
+        </div>
+    </div>
+
+    <div class="container">
+        <ul>
+            {% for movie in object_list %}
+            <li>{{movie}}</li>
+            {% endfor %}
+        </ul>
+    </div>
+</main>
+{% endblock %}

vrobbler/urls.py

@@ -1,12 +1,11 @@
+import scrobbles.views as scrobbles_views
 from django.conf import settings
 from django.conf.urls.static import static
 from django.contrib import admin
 from django.urls import include, path
-from rest_framework import routers
-from scrobbles.views import RecentScrobbleList
-from videos import urls as video_urls
-
 from scrobbles import urls as scrobble_urls
+from music import urls as music_urls
+from videos import urls as video_urls
 
 urlpatterns = [
     path("admin/", admin.site.urls),
@@ -15,8 +14,21 @@ urlpatterns = [
     # path("movies/", include(movies, namespace="movies")),
     # path("shows/", include(shows, namespace="shows")),
     path("api/v1/scrobbles/", include(scrobble_urls, namespace="scrobbles")),
+    path(
+        'manual/imdb/',
+        scrobbles_views.ManualScrobbleView.as_view(),
+        name='imdb-manual-scrobble',
+    ),
+    path(
+        'manual/audioscrobbler/',
+        scrobbles_views.AudioScrobblerImportCreateView.as_view(),
+        name='audioscrobbler-file-upload',
+    ),
+    path("", include(music_urls, namespace="music")),
     path("", include(video_urls, namespace="videos")),
-    path("", RecentScrobbleList.as_view(), name="home"),
+    path(
+        "", scrobbles_views.RecentScrobbleList.as_view(), name="vrobbler-home"
+    ),
 ]
 
 if settings.DEBUG: