Bump version to 0.13.2

This reverts commit c00343abfe.

.coveragerc

@@ -0,0 +1,7 @@
+[run]
+omit=
+  vrobbler/wsgi.py
+  vrobbler/asgi.py
+  vrobbler/cli.py
+  *admin.py
+  migrations/*

.drone.yml

@@ -8,21 +8,40 @@ name: run_tests
 
 steps:
   # Run tests against Python/Flask engine backend (with pytest)
-  - name: django_tests
+  - name: pytest with coverage
     image: python:3.10.4
     commands:
       # Install dependencies
-      - cp vrobbler.conf.example vrobbler.conf
+      - cp vrobbler.conf.test vrobbler.conf
       - pip install poetry
+      - poetry config repositories.unblink "https://pypi.unbl.ink/pypi/simple"
       - poetry install
       # Start with a fresh database (which is already running as a service from Drone)
-      - poetry run python manage.py test
+      - poetry run pytest --cov-report term:skip-covered --cov=vrobbler tests
     environment:
       VROBBLER_DATABASE_URL: sqlite:///test.db
     volumes:
       # Mount pip cache from host
       - name: pip_cache
         path: /root/.cache/pip
+  - name: deploy
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - vrobbler.service
+      username: root
+      ssh_key:
+        from_secret: ssh_key
+      command_timeout: 2m
+      script:
+        - pip uninstall -y vrobbler
+        - pip install git+https://code.unbl.ink/secstate/vrobbler.git@main
+        - vrobbler migrate
+        - vrobbler collectstatic --noinput
+        - immortalctl restart celery && immortalctl restart vrobbler
+    when:
+      branch:
+      - main
 volumes:
   - name: docker
     host:

.gitignore

@@ -1,4 +1,5 @@
 db.sqlite3
 vrobbler.conf
-/media/
-/dist/
+media/
+dist/
+.coverage

Procfile

@@ -0,0 +1,2 @@
+web: python manage.py runserver 0.0.0.0:8014
+worker: celery -A vrobbler worker -l DEBUG

README.md

@@ -1,4 +1,7 @@
-#+title: Readme
+Vrobbler
+========
+
+[![Build Status](https://ci.unbl.ink/api/badges/secstate/vrobbler/status.svg?ref=refs/heads/main)](https://ci.unbl.ink/secstate/vrobbler)
 
 Vrobbler is a pretty simple Django-powered web app for scrobbling video plays from you favorite Jellyfin installation.
 

envrc.sample

@@ -0,0 +1,4 @@
+export ENV_PATH=$(poetry env info --path)
+source "${ENV_PATH}/bin/activate"
+
+export PYPI_PASSWORD="$(pass personal/apikey/pypi)"

manage.py

@@ -6,7 +6,7 @@ import sys
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'vrobbler.settings')
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vrobbler.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:
@@ -18,5 +18,5 @@ def main():
     execute_from_command_line(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vrobbler"
-version = "0.2.2"
+version = "0.11.12"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 
@@ -16,7 +16,7 @@ djangorestframework = "^3.13.1"
 Markdown = "^3.3.6"
 django-filter = "^21.1"
 Pillow = "^9.0.1"
-psycopg2 = {version = "^2.9.3", extras = ["production"]}
+psycopg2 = "^2.9.3"
 dj-database-url = "^0.5.0"
 django-mathfilters = "^1.0.0"
 django-allauth = "^0.50.0"
@@ -26,28 +26,47 @@ django-taggit = "^2.1.0"
 django-markdownify = "^0.9.1"
 gunicorn = "^20.1.0"
 django-simple-history = "^3.1.1"
-whitenoise = "^6.3.0"
+musicbrainzngs = "^0.7.1"
+cinemagoer = "^2022.12.27"
+pysportsdb = "^0.1.0"
+pytz = "^2022.7.1"
+django-redis = "^5.2.0"
+pylast = "^5.1.0"
+django-encrypted-field = "^1.0.5"
+celery = "^5.2.7"
+honcho = "^1.1.0"
+howlongtobeatpy = "^1.0.5"
+beautifulsoup4 = "^4.11.2"
+django-storages = "^1.13.2"
+boto3 = "^1.26.98"
+stream-sqlite = "^0.0.41"
 
 [tool.poetry.dev-dependencies]
 Werkzeug = "2.0.3"
 black = "^22.3"
-freezegun = "^1.2"
+coverage = "^7.0.5"
 mypy = "^0.961"
 pytest = "^7.1"
 pytest-black = "^0.3.12"
 pytest-cov = "^3.0"
+pytest-django = "^4.5.2"
 pytest-flake8 = "^1.1"
 pytest-isort = "^3.0"
 pytest-runner = "^6.0"
 pytest-selenium = "^2.0.1"
+time-machine = "^2.9.0"
 types-pytz = "^2022.1"
 types-requests = "^2.27"
-types-freezegun = "^1.1"
 bandit = "^1.7.4"
 
+[tool.pytest.ini_options]
+minversion = "6.0"
+addopts = "-ra -q"
+testpaths = ["tests"]
+DJANGO_SETTINGS_MODULE='vrobbler.settings'
+
 [tool.black]
 line-length = 79
-skip-string-normalization = true
 target-version = ["py39", "py310"]
 include = ".py$"
 exclude = "migrations"

templates/base.html

@@ -1,25 +0,0 @@
-<!doctype html>
-<html class="no-js" lang="">
-    <head>
-        <meta charset="utf-8">
-        <meta http-equiv="x-ua-compatible" content="ie=edge">
-        <title>Untitled</title>
-        <meta name="description" content="">
-        <meta name="viewport" content="width=device-width, initial-scale=1">
-
-        <link rel="apple-touch-icon" href="/apple-touch-icon.png">
-        <!-- Place favicon.ico in the root directory -->
-
-    </head>
-    <body>
-        <!--[if lt IE 8]>
-            <p class="browserupgrade">
-            You are using an <strong>outdated</strong> browser. Please
-            <a href="http://browsehappy.com/">upgrade your browser</a> to improve
-            your experience.
-            </p>
-        <![endif]-->
-
-
-    </body>
-</html>

tests/podcasts_tests/test_scrapers.py

@@ -0,0 +1,28 @@
+from vrobbler.apps.podcasts.scrapers import scrape_data_from_google_podcasts
+
+expected_desc = (
+    "NPR's Up First is the news you need to start your day. "
+    "The three biggest stories of the day, with reporting and analysis "
+    "from NPR News — in 10 minutes. Available weekdays by 6 a.m. ET, "
+    "with hosts Leila Fadel, Steve Inskeep, Michel Martin and A Martinez. "
+    "Also available on Saturdays by 8 a.m. ET, with Ayesha Rascoe and "
+    "Scott Simon. On Sundays, hear a longer exploration behind the "
+    "headlines with Rachel Martin, available by 8 a.m. ET. Subscribe "
+    "and listen, then support your local NPR station at donate.npr.org.  "
+    "Support NPR's reporting by subscribing to Up First+ and unlock "
+    "sponsor-free listening. Learn more at plus.npr.org/UpFirst"
+)
+
+expected_img_url = "https://encrypted-tbn2.gstatic.com/images?q=tbn:ANd9GcT-PqtK-bauo8wm8dBE__SVGArlvfBYY8rqxr2kA5UwjKzEx8c"
+expected_google_url = "https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzUxMDMxOC9wb2RjYXN0LnhtbA"
+
+
+def test_get_not_allowed_from_mopidy():
+    query = "Up First"
+    result_dict = scrape_data_from_google_podcasts(query)
+
+    assert result_dict["title"] == query
+    assert result_dict["description"] == expected_desc
+    assert result_dict["image_url"] == expected_img_url
+    assert result_dict["producer"] == "NPR"
+    assert result_dict["google_url"] == expected_google_url

tests/scrobbles_tests/conftest.py

@@ -0,0 +1,83 @@
+import json
+import pytest
+
+from rest_framework.authtoken.models import Token
+from django.contrib.auth import get_user_model
+
+User = get_user_model()
+
+
+class MopidyRequest:
+    name = "Same in the End"
+    artist = "Sublime"
+    album = "Sublime"
+    track_number = 4
+    run_time_ticks = 156604
+    run_time = "156"
+    playback_time_ticks = 15045
+    musicbrainz_track_id = "54214d63-5adf-4909-87cd-c65c37a6d558"
+    musicbrainz_album_id = "03b864cd-7761-314c-a892-05a89ddff00d"
+    musicbrainz_artist_id = "95f5b748-d370-47fe-85bd-0af2dc450bc0"
+    mopidy_uri = "local:track:Sublime%20-%20Sublime/Disc%201%20-%2004%20-%20Same%20in%20the%20End.mp3"  # noqa
+    status = "resumed"
+
+    def __init__(self, **kwargs):
+        self.request_data = {
+            "name": kwargs.get("name", self.name),
+            "artist": kwargs.get("artist", self.artist),
+            "album": kwargs.get("album", self.album),
+            "track_number": int(kwargs.get("track_number", self.track_number)),
+            "run_time_ticks": int(
+                kwargs.get("run_time_ticks", self.run_time_ticks)
+            ),
+            "run_time": int(kwargs.get("run_time", self.run_time)),
+            "playback_time_ticks": int(
+                kwargs.get("playback_time_ticks", self.playback_time_ticks)
+            ),
+            "musicbrainz_track_id": kwargs.get(
+                "musicbrainz_track_id", self.musicbrainz_track_id
+            ),
+            "musicbrainz_album_id": kwargs.get(
+                "musicbrainz_album_id", self.musicbrainz_album_id
+            ),
+            "musicbrainz_artist_id": kwargs.get(
+                "musicbrainz_artist_id", self.musicbrainz_artist_id
+            ),
+            "mopidy_uri": kwargs.get("mopidy_uri", self.mopidy_uri),
+            "status": kwargs.get("status", self.status),
+        }
+
+    def __eq__(self, other):
+        for key in self.request_data.keys():
+            if self.request_data[key] != getattr(self, key):
+                return False
+        return True
+
+    @property
+    def request_json(self):
+        return json.dumps(self.request_data)
+
+
+@pytest.fixture
+def valid_auth_token():
+    user = User.objects.create(email="test@exmaple.com")
+    return Token.objects.create(user=user).key
+
+
+@pytest.fixture
+def mopidy_track_request_data():
+    return MopidyRequest().request_json
+
+
+@pytest.fixture
+def mopidy_track_diff_album_request_data(**kwargs):
+    mb_album_id = "0c56c457-afe1-4679-baab-759ba8dd2a58"
+    return MopidyRequest(
+        album="Gold", musicbrainz_album_id=mb_album_id
+    ).request_json
+
+
+@pytest.fixture
+def mopidy_podcast_request_data():
+    mopidy_uri = "local:podcast:Up%20First/2022-01-01%20Up%20First.mp3"
+    return MopidyRequest(mopidy_uri=mopidy_uri).request_json

tests/scrobbles_tests/test_aggregators.py

@@ -0,0 +1,102 @@
+from datetime import datetime, timedelta
+
+import pytest
+import time_machine
+from django.contrib.auth import get_user_model
+from django.urls import reverse
+from django.utils import timezone
+from music.aggregators import live_charts, scrobble_counts, week_of_scrobbles
+from profiles.models import UserProfile
+from scrobbles.models import Scrobble
+
+
+def build_scrobbles(client, request_data, num=7, spacing=2):
+    url = reverse("scrobbles:mopidy-webhook")
+    user = get_user_model().objects.create(username="Test User")
+    UserProfile.objects.create(user=user, timezone="US/Eastern")
+    for i in range(num):
+        client.post(url, request_data, content_type="application/json")
+        s = Scrobble.objects.last()
+        s.user = user
+        s.timestamp = timezone.now() - timedelta(days=i * spacing)
+        s.played_to_completion = True
+        s.save()
+
+
+@pytest.mark.django_db
+@time_machine.travel(datetime(2022, 3, 4, 1, 24))
+def test_scrobble_counts_data(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data)
+    user = get_user_model().objects.first()
+    count_dict = scrobble_counts(user)
+    assert count_dict == {
+        "alltime": 7,
+        "month": 2,
+        "today": 1,
+        "week": 3,
+        "year": 7,
+    }
+
+
+@pytest.mark.django_db
+def test_week_of_scrobbles_data(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    week = week_of_scrobbles(user)
+    assert list(week.values()) == [1, 1, 1, 1, 1, 1, 1]
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_day(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = live_charts(user)
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_week(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = live_charts(user, chart_period="week")
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_month(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = live_charts(user, chart_period="month")
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top_tracks_by_year(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = live_charts(user, chart_period="year")
+    assert tops[0].title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_top__artists_by_week(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = live_charts(user, chart_period="week", media_type="Artist")
+    assert tops[0].name == "Sublime"
+
+
+@pytest.mark.django_db
+def test_top__artists_by_month(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = live_charts(user, chart_period="month", media_type="Artist")
+    assert tops[0].name == "Sublime"
+
+
+@pytest.mark.django_db
+def test_top__artists_by_year(client, mopidy_track_request_data):
+    build_scrobbles(client, mopidy_track_request_data, 7, 1)
+    user = get_user_model().objects.first()
+    tops = live_charts(user, chart_period="year", media_type="Artist")
+    assert tops[0].name == "Sublime"

tests/scrobbles_tests/test_views.py

@@ -0,0 +1,100 @@
+import pytest
+from django.urls import reverse
+from music.models import Track
+from podcasts.models import Episode
+from scrobbles.models import Scrobble
+
+
+@pytest.mark.django_db
+def test_get_not_allowed_from_mopidy(client, valid_auth_token):
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.get(url, headers=headers)
+    assert response.status_code == 405
+
+
+@pytest.mark.django_db
+def test_bad_mopidy_request_data(client, valid_auth_token):
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(url, headers)
+    assert response.status_code == 400
+    assert (
+        response.data["detail"]
+        == "JSON parse error - Expecting value: line 1 column 1 (char 0)"
+    )
+
+
+@pytest.mark.django_db
+def test_scrobble_mopidy_track(
+    client, mopidy_track_request_data, valid_auth_token
+):
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(
+        url,
+        mopidy_track_request_data,
+        content_type="application/json",
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 1}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.__class__ == Track
+    assert scrobble.media_obj.title == "Same in the End"
+
+
+@pytest.mark.skip(reason="API is unstable")
+@pytest.mark.django_db
+def test_scrobble_mopidy_same_track_different_album(
+    client,
+    mopidy_track_request_data,
+    mopidy_track_diff_album_request_data,
+    valid_auth_token,
+):
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(
+        url,
+        mopidy_track_request_data,
+        content_type="application/json",
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 1}
+    scrobble = Scrobble.objects.last()
+    assert scrobble.media_obj.album.name == "Sublime"
+
+    response = client.post(
+        url,
+        mopidy_track_diff_album_request_data,
+        content_type="application/json",
+    )
+
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 2}
+    scrobble = Scrobble.objects.last()
+    assert scrobble.media_obj.__class__ == Track
+    assert scrobble.media_obj.album.name == "Gold"
+    assert scrobble.media_obj.title == "Same in the End"
+
+
+@pytest.mark.django_db
+def test_scrobble_mopidy_podcast(
+    client, mopidy_podcast_request_data, valid_auth_token
+):
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(
+        url,
+        mopidy_podcast_request_data,
+        content_type="application/json",
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 1}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.__class__ == Episode
+    assert scrobble.media_obj.title == "Up First"

tests/videos_tests/test_imdb.py

@@ -0,0 +1,11 @@
+import pytest
+
+from videos.imdb import lookup_video_from_imdb
+
+
+@pytest.mark.skip(reason="Need to sort out third party API testing")
+def test_lookup_imdb_bad_id(caplog):
+    data = lookup_video_from_imdb("3409324")
+    assert data is None
+    assert caplog.records[0].levelname == "WARNING"
+    assert caplog.records[0].msg == "IMDB ID should begin with 'tt' 3409324"

todos.org

@@ -0,0 +1,446 @@
+#+title: TODOs
+
+A fun way to keep track of things in the project to fix or improve.
+
+* Version 1.0.0
+** DONE Add a "stop_timestamp" so we don't rely on content length :improvement:scrobbling:
+CLOSED: [2023-04-02 Sun 23:58]
+
+Essentially, we currently have the timestamp as when the content began
+scrobbling and then calculate the finish time from the length of the content.
+This works pretty well because we know how long most things are.
+
+But in some cases, sports events or long podcasts, we may start mid-way through
+an event or finish halfway through but still want to mark it as done. In these
+cases, knowing the finish time could be useful, especially when interfacing with
+other scrobblers which may have different definitions of when a scrobble
+finishes or started.
+** DONE Fix bug with Various Artist albums being labeled with first artist as album artist :scrobbling:bug:music:
+CLOSED: [2023-03-27 Mon 20:18]
+:LOGBOOK:
+CLOCK: [2023-03-26 Sun 22:01]--[2023-03-27 Mon 01:07] =>  3:06
+:END:
+** DONE Fix bug with weekly aggregator being blank on Sundays :aggregators:music:bug:
+CLOSED: [2023-03-26 Sun 13:52]
+** DONE Fix KoReader scrobbling to use pages rather than time of last read :scrobbling:books:improvement:
+CLOSED: [2023-03-26 Sun 13:51]
+:LOGBOOK:
+CLOCK: [2023-03-26 Sun 13:11]--[2023-03-26 Sun 13:51] =>  0:40
+:END:
+** DONE [#A] Add django-storage to store files on S3 :settings:improvement:
+CLOSED: [2023-03-24 Fri 14:46]
+:LOGBOOK:
+CLOCK: [2023-03-24 Fri 10:47]--[2023-03-24 Fri 14:46] =>  3:59
+CLOCK: [2023-03-24 Fri 10:36]--[2023-03-24 Fri 10:40] =>  0:04
+:END:
+** DONE Fix vrobbler settings not using booleans :settings:bug:
+CLOSED: [2023-03-24 Fri 10:45]
+:LOGBOOK:
+CLOCK: [2023-03-24 Fri 10:40]--[2023-03-24 Fri 10:46] =>  0:06
+:END:
+** DONE Update weekly live chart to be 7-day continuous rather than weekly :views:bug:
+CLOSED: [2023-03-24 Fri 00:31]
+The live view will be blank every Monday, no reason to tie it to a day of the
+week. It should be "the last 7 days"
+** DONE [#B] Implement a detail view for TV shows :improvement:views:
+CLOSED: [2023-03-22 Wed 17:05]
+** DONE [#B] Implement a detail view for Movies :improvement:views:
+CLOSED: [2023-03-22 Wed 17:05]
+** DONE Add "service provider" to TV Series, and use that for source when available :bug:scrobbling:
+CLOSED: [2023-03-22 Wed 17:04]
+** DONE Add view for long-play content (books, video games) to restart them :views:improvement:
+CLOSED: [2023-03-22 Wed 17:01]
+** DONE Add live chart view like Maloja :improvement:views:
+CLOSED: [2023-03-07 Tue 11:13]
+** DONE [#C] Figure out how to add to web-scrobbler :improvement:scrobbling:
+CLOSED: [2023-03-22 Wed 17:06]
+
+An example:
+https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
+
+This is actually going to be moot because we can import from LastFM, and
+web-scrobbler integrates well with LastFM. The only thing to think through here
+now is what to do with all the garbage web-scrobbler sometimes pushes to LastFM
+from Youtube (all videos get pushed, sigh).
+
+* Version 0.11.4
+** DONE Add rudimentary video game scrobbling :improvement:content:videogames:
+CLOSED: [2023-03-07 Tue 11:11]
+** DONE Add ability to scrobble from KOReader statistics files :improvement:books:content:
+CLOSED: [2023-03-07 Tue 11:11]
+
+** DONE [#A] Fix fetching artwork without release group :bug:
+CLOSED: [2023-01-29 Sun 14:27]
+
+When we get artwork from Musicbrianz, and it's not found, we should check for
+release groups as well. This will stop issues with missing artwork because of
+obscure MB release matches.
+
+** DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion percent :bug:
+CLOSED: [2023-01-30 Mon 18:31]
+:LOGBOOK:
+CLOCK: [2023-01-30 Mon 18:00]--[2023-01-30 Mon 18:31] =>  0:31
+:END:
+
+If we play music from Jellyfin and the track reaches 90% completion, the
+scrobbling goes crazy and starts creating new scrobbles with every update.
+
+The cause is pretty simple, but the solution is hard. We want to mark a scrobble
+as complete for the following conditions:
+
+- Play stopped and percent played beyond 90%
+- Play completely finished
+
+But if we keep listening beyond 90, we should basically ignore updates (or just
+update the existing scrobble)
+** DONE [#A] Add support for Audioscrobbler tab-separated file uploads :improvement:
+CLOSED: [2023-02-03 Fri 16:52]
+
+An example of the format:
+#+begin_src csv
+,
+#AUDIOSCROBBLER/1.1
+#TZ/UNKNOWN
+#CLIENT/Rockbox sansaclipplus $Revision$
+75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494944	64ff5f53-d187-4512-827e-7606c69e66ff
+75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494990	64ff5f53-d187-4512-827e-7606c69e66ff
+311	311	Down	1	173	S	1740495003	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Down	1	173	L	1740495049	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Down	1	173	L	1740495113	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Random	2	187	S	1740495190	530c09f3-46fe-4d90-b11f-7b63bcb4b373
+311	311	Random	2	187	L	1740495194	530c09f3-46fe-4d90-b11f-7b63bcb4b373
+311	311	Jackolantern’s Weather	3	204	L	1740495382	cc3b2dec-5d99-47ea-8930-20bf258be4ea
+311	311	All Mixed Up	4	182	L	1740495586	980a78b5-5bdd-4f50-9e3a-e13261e2817b
+311	311	Hive	5	179	L	1740495768	18f6dc98-d3a2-4f81-b967-97359d14c68c
+311	311	Guns (Are for Pussies)	6	137	L	1740495948	5e97ed9f-c8cc-4282-9cbe-f8e17aee5128
+311	311	Misdirected Hostility	7	179	S	1740496085	61ff2c1a-fc9c-44c3-8da1-5e50a44245af
+,
+#+end_src
+** DONE [#B] Allow scrobbling music without MB IDs by grabbing them before scrobble :improvement:
+CLOSED: [2023-02-17 Fri 00:10]
+
+This would allow a few nice flows. One, you'd be able to record the play of an
+entire album by just dropping the muscibrainz_id in. This could be helpful for
+offline listening. It would also mean bad metadata from mopidy would not break
+scrobbling.
+** DONE When updating musicbrainz IDs, clear and run fetch artwrok :improvement:
+CLOSED: [2023-02-17 Fri 00:11]
+** DONE [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
+CLOSED: [2023-03-07 Tue 11:09]
+
+Given a UUID from musicbrainz, we should be able to scrobble an album or
+individual track.
+
+** DONE [#C] Implement keeping track of week/month/year chart-toppers :improvement:
+CLOSED: [2023-03-07 Tue 11:10]
+:LOGBOOK:
+CLOCK: [2023-01-30 Mon 16:30]--[2023-01-30 Mon 18:00] =>  1:30
+:END:
+
+Maloja does this cool thing where artists and tracks get recorded as the top
+track of a given week, month or year. They get gold, silver or bronze stars for
+their place in the time period.
+
+I could see this being implemented as a separate Chart table which gets
+populated at the end of a time period and has a start and end date that defines
+a period, along with a one, two, three instance.
+
+Of course, it could also be a data model without a table, where it runs some fun
+calculations, stores it's values in Redis as a long-term lookup table and just
+has to re-populate when the server restarts.
+* Backlog
+** TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :utility:improvement:
+*** Example payloads from mopidy-webhooks
+**** Podcast playback ended
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_ended",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 13,
+            "track": {
+                "__model__": "Track",
+                "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
+                "name": "Wolf warriors",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "name": "The Economist"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "name": "The Prince",
+                    "date": "2022"
+                },
+                "genre": "Blues",
+                "date": "2022",
+                "length": 2437778,
+                "bitrate": 127988
+            }
+        },
+        "time_position": 3290
+    }
+}
+#+end_src
+**** Podcast playback state changes
+#+begin_src json
+{
+    "type": "event",
+    "event": "playback_state_changed",
+    "data": {
+        "old_state": "paused",
+        "new_state": "playing"
+    }
+}
+#+end_src
+
+#+begin_src json
+{
+    "type": "event",
+    "event": "playback_state_changed",
+    "data": {
+        "old_state": "stopped",
+        "new_state": "playing"
+    }
+}
+#+end_src
+**** Podcast playback started
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_started",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 13,
+            "track": {
+                "__model__": "Track",
+                "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
+                "name": "Wolf warriors",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "name": "The Economist"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "name": "The Prince",
+                    "date": "2022"
+                },
+                "genre": "Blues",
+                "date": "2022",
+                "length": 2437778,
+                "bitrate": 127988
+            }
+        }
+    }
+}
+#+end_src
+**** Podcast playback paused
+#+begin_src  json
+{
+    "type": "status",
+    "data": {
+        "state": "paused",
+        "current_track": {
+            "__model__": "Track",
+            "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
+            "name": "Wolf warriors",
+            "artists": [
+                {
+                    "__model__": "Artist",
+                    "name": "The Economist"
+                }
+            ],
+            "album": {
+                "__model__": "Album",
+                "name": "The Prince",
+                "date": "2022"
+            },
+            "genre": "Blues",
+            "date": "2022",
+            "length": 2437778,
+            "bitrate": 127988
+        },
+        "time_position": 2350
+    }
+}
+
+#+end_src
+**** Track playback started
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_started",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 14,
+            "track": {
+                "__model__": "Track",
+                "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
+                "name": "Supermassive Black Hole",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
+                        "name": "Muse",
+                        "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
+                    "name": "Twilight: Original Motion Picture Soundtrack",
+                    "artists": [
+                        {
+                            "__model__": "Artist",
+                            "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
+                            "name": "Various Artists",
+                            "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
+                        }
+                    ],
+                    "num_tracks": 12,
+                    "num_discs": 1,
+                    "date": "2008-11-04",
+                    "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
+                },
+                "composers": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
+                        "name": "Matt Bellamy"
+                    }
+                ],
+                "genre": "Rock",
+                "track_no": 1,
+                "disc_no": 1,
+                "date": "2008-11-04",
+                "length": 211121,
+                "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
+                "last_modified": 1672712949510
+            }
+        }
+    }
+}
+#+end_src
+**** Track playback in progress
+#+begin_src json
+{
+    "type": "status",
+    "data": {
+        "state": "playing",
+        "current_track": {
+            "__model__": "Track",
+            "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
+            "name": "Supermassive Black Hole",
+            "artists": [
+                {
+                    "__model__": "Artist",
+                    "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
+                    "name": "Muse",
+                    "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
+                }
+            ],
+            "album": {
+                "__model__": "Album",
+                "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
+                "name": "Twilight: Original Motion Picture Soundtrack",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
+                        "name": "Various Artists",
+                        "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
+                    }
+                ],
+                "num_tracks": 12,
+                "num_discs": 1,
+                "date": "2008-11-04",
+                "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
+            },
+            "composers": [
+                {
+                    "__model__": "Artist",
+                    "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
+                    "name": "Matt Bellamy"
+                }
+            ],
+            "genre": "Rock",
+            "track_no": 1,
+            "disc_no": 1,
+            "date": "2008-11-04",
+            "length": 211121,
+            "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
+            "last_modified": 1672712949510
+        },
+        "time_position": 17031
+    }
+}
+#+end_src
+**** Track event playback paused
+#+begin_src json
+{
+    "type": "event",
+    "event": "track_playback_paused",
+    "data": {
+        "tl_track": {
+            "__model__": "TlTrack",
+            "tlid": 14,
+            "track": {
+                "__model__": "Track",
+                "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
+                "name": "Supermassive Black Hole",
+                "artists": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
+                        "name": "Muse",
+                        "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
+                    }
+                ],
+                "album": {
+                    "__model__": "Album",
+                    "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
+                    "name": "Twilight: Original Motion Picture Soundtrack",
+                    "artists": [
+                        {
+                            "__model__": "Artist",
+                            "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
+                            "name": "Various Artists",
+                            "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
+                        }
+                    ],
+                    "num_tracks": 12,
+                    "num_discs": 1,
+                    "date": "2008-11-04",
+                    "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
+                },
+                "composers": [
+                    {
+                        "__model__": "Artist",
+                        "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
+                        "name": "Matt Bellamy"
+                    }
+                ],
+                "genre": "Rock",
+                "track_no": 1,
+                "disc_no": 1,
+                "date": "2008-11-04",
+                "length": 211121,
+                "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
+                "last_modified": 1672712949510
+            }
+        },
+        "time_position": 67578
+    }
+}
+#+end_src
+** TODO [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :utililty:improvement:
+** TODO What to do with Youtube videos from LastFM and web-scrobbler :bug:source:lastfm:
+** TODO Fix bug in Jellyfin scrobbles that spam more scrobbles after completion :scrobbling:videos:bug:
+** TODO Fix bug in podcast scrobbling where a second scrobble is created after completion :scrobbling:podcasts:bug:

vrobbler.conf.example

@@ -1,11 +1,26 @@
 # You can use this file to set environment variables for your local setup
 #
 VROBBLER_DEBUG=True
-VROBBLER_JSON_LOGGING=True
 VROBBLER_LOG_LEVEL="DEBUG"
+VROBBLER_JSON_LOGGING=True
 VROBBLER_MEDIA_ROOT = "/media/"
-VROBBLER_TMDB_API_KEY = "KEY"
-VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True
+VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=False
 
-VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"
-VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0"
+VROBBLER_USE_S3=False
+# You may also need to set these in your environment
+AWS_S3_ACCESS_KEY_ID=""
+AWS_S3_SECRET_ACCESS_KEY=""
+AWS_S3_CUSTOM_DOMAIN="https://minio.dev/"
+
+# API keys
+VROBBLER_TMDB_API_KEY = "<key>"
+VROBBLER_LASTFM_API_KEY = "<key>"
+VROBBLER_LASTFM_SECRET_KEY = "<key>"
+VROBBLER_THESPORTSDB_API_KEY="<key>"
+VROBBLER_THEAUDIODB_API_KEY="<key>"
+VROBBLER_IGDB_CLIENT_ID="<id>"
+VROBBLER_IGDB_CLIENT_SECRET="<key>"
+
+# Storages
+# VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"
+# VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0"

vrobbler.conf.test

@@ -0,0 +1,8 @@
+# Local configuration for Emus
+
+VROBBLER_DUMP_REQUEST_DATA=True
+VROBBLER_LOG_TO_CONSOLE=True
+VROBBLER_DEBUG=True
+VROBBLER_LOG_LEVEL="DEBUG"
+VROBBLER_MEDIA_ROOT = "/tmp/media/"
+VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True

vrobbler/__init__.py

@@ -0,0 +1,5 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ("celery_app",)

vrobbler/apps/books/admin.py

@@ -0,0 +1,42 @@
+from django.contrib import admin
+
+from books.models import Author, Book, Page
+
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(Author)
+class AuthorAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "name",
+        "openlibrary_id",
+        "bio",
+        "wikipedia_url",
+    )
+    ordering = ("-created",)
+    search_fields = ("name",)
+
+
+@admin.register(Page)
+class PageAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_filter = ("book",)
+    ordering = ("book", "number")
+
+
+@admin.register(Book)
+class BookAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "title",
+        "isbn",
+        "first_publish_year",
+        "pages",
+        "openlibrary_id",
+    )
+    search_fields = ("name",)
+    ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/books/api/serializers.py

@@ -0,0 +1,14 @@
+from books.models import Author, Book
+from rest_framework import serializers
+
+
+class AuthorSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Author
+        fields = "__all__"
+
+
+class BookSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Book
+        fields = "__all__"

vrobbler/apps/books/api/views.py

@@ -0,0 +1,19 @@
+from rest_framework import permissions, viewsets
+
+from books.api.serializers import (
+    AuthorSerializer,
+    BookSerializer,
+)
+from books.models import Author, Book
+
+
+class AuthorViewSet(viewsets.ModelViewSet):
+    queryset = Author.objects.all().order_by("-created")
+    serializer_class = AuthorSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class BookViewSet(viewsets.ModelViewSet):
+    queryset = Book.objects.all().order_by("-created")
+    serializer_class = BookSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/books/koreader.py

@@ -0,0 +1,259 @@
+import codecs
+import logging
+import os
+import re
+import sqlite3
+from datetime import datetime
+from enum import Enum
+from typing import Iterable, List
+
+import pytz
+import requests
+from books.models import Author, Book, Page
+from books.openlibrary import get_author_openlibrary_id
+from django.db.models import Sum
+from pylast import httpx, tempfile
+from scrobbles.models import Scrobble
+from stream_sqlite import stream_sqlite
+
+logger = logging.getLogger(__name__)
+
+
+class KoReaderBookColumn(Enum):
+    ID = 0
+    TITLE = 1
+    AUTHORS = 2
+    NOTES = 3
+    LAST_OPEN = 4
+    HIGHLIGHTS = 5
+    PAGES = 6
+    SERIES = 7
+    LANGUAGE = 8
+    MD5 = 9
+    TOTAL_READ_TIME = 10
+    TOTAL_READ_PAGES = 11
+
+
+class KoReaderPageStatColumn(Enum):
+    ID_BOOK = 0
+    PAGE = 1
+    START_TIME = 2
+    DURATION = 3
+    TOTAL_PAGES = 4
+
+
+def _sqlite_bytes(sqlite_url):
+    with httpx.stream("GET", sqlite_url) as r:
+        yield from r.iter_bytes(chunk_size=65_536)
+
+
+def get_book_map_from_sqlite(rows: Iterable) -> dict:
+    """Given an interable of sqlite rows from the books table, lookup existing
+    books, create ones that don't exist, and return a mapping of koreader IDs to
+    primary key IDs for page creation.
+
+    """
+    book_id_map = {}
+
+    for book_row in rows:
+        book = Book.objects.filter(
+            koreader_md5=book_row[KoReaderBookColumn.MD5.value]
+        ).first()
+
+        if not book:
+            book, created = Book.objects.get_or_create(
+                title=book_row[KoReaderBookColumn.TITLE.value]
+            )
+
+            if created:
+                total_pages = book_row[KoReaderBookColumn.PAGES.value]
+                run_time = total_pages * book.AVG_PAGE_READING_SECONDS
+                ko_authors = book_row[
+                    KoReaderBookColumn.AUTHORS.value
+                ].replace("\n", ", ")
+                # Strip middle initials, OpenLibrary often fails with these
+                ko_authors = re.sub(" [A-Z]. ", " ", ko_authors)
+                book_dict = {
+                    "title": book_row[KoReaderBookColumn.TITLE.value],
+                    "pages": total_pages,
+                    "koreader_md5": book_row[KoReaderBookColumn.MD5.value],
+                    "koreader_id": int(book_row[KoReaderBookColumn.ID.value]),
+                    "koreader_authors": ko_authors,
+                    "run_time_seconds": run_time,
+                }
+                Book.objects.filter(pk=book.id).update(**book_dict)
+
+                # Add authors
+                authors = ko_authors.split(", ")
+                author_list = []
+                for author_str in authors:
+                    logger.debug(f"Looking up author {author_str}")
+                    if author_str == "N/A":
+                        continue
+
+                    author, created = Author.objects.get_or_create(
+                        name=author_str
+                    )
+                    if created:
+                        author.openlibrary_id = get_author_openlibrary_id(
+                            author_str
+                        )
+                        author.save(update_fields=["openlibrary_id"])
+                        author.fix_metadata()
+                        logger.debug(f"Created author {author}")
+                    book.authors.add(author)
+
+                # This will try to fix metadata by looking it up on OL
+                book.fix_metadata()
+
+        book.refresh_from_db()
+        total_seconds = 0
+        if book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]:
+            total_seconds = book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]
+
+        book_id_map[book_row[KoReaderBookColumn.ID.value]] = (
+            book.id,
+            total_seconds,
+        )
+
+    return book_id_map
+
+
+def build_scrobbles_from_pages(
+    rows: Iterable, book_id_map: dict, user_id: int
+) -> List[Scrobble]:
+    new_scrobbles = []
+
+    new_scrobbles = []
+    pages_found = []
+    book_read_time_map = {}
+    for page_row in rows:
+        koreader_id = page_row[KoReaderPageStatColumn.ID_BOOK.value]
+        page_number = page_row[KoReaderPageStatColumn.PAGE.value]
+        ts = page_row[KoReaderPageStatColumn.START_TIME.value]
+        book_id = book_id_map[koreader_id][0]
+        book_read_time_map[book_id] = book_id_map[koreader_id][1]
+
+        page, page_created = Page.objects.get_or_create(
+            book_id=book_id, number=page_number, user_id=user_id
+        )
+        if page_created:
+            page.start_time = datetime.utcfromtimestamp(ts).replace(
+                tzinfo=pytz.utc
+            )
+            page.duration_seconds = page_row[
+                KoReaderPageStatColumn.DURATION.value
+            ]
+            page.save(update_fields=["start_time", "duration_seconds"])
+        pages_found.append(page)
+
+    playback_position_seconds = 0
+    for page in set(pages_found):
+        # Add up page seconds to set the aggregate time of all pages to reading time
+        playback_position_seconds = (
+            playback_position_seconds + page.duration_seconds
+        )
+        if page.is_scrobblable:
+            # Check to see if a scrobble with this timestamp, book and user already exists
+            scrobble = Scrobble.objects.filter(
+                timestamp=page.start_time,
+                book_id=page.book_id,
+                user_id=user_id,
+            ).first()
+            if not scrobble:
+                logger.debug(
+                    f"Queueing scrobble for {page.book}, page {page.number}"
+                )
+                new_scrobble = Scrobble(
+                    book_id=page.book_id,
+                    user_id=user_id,
+                    source="KOReader",
+                    timestamp=page.start_time,
+                    played_to_completion=True,
+                    playback_position_seconds=playback_position_seconds,
+                    in_progress=False,
+                    book_pages_read=page.number,
+                    long_play_complete=False,
+                )
+                new_scrobbles.append(new_scrobble)
+            # After setting a scrobblable page, reset our accumulator
+            playback_position_seconds = 0
+    return new_scrobbles
+
+
+def enrich_koreader_scrobbles(scrobbles: list) -> None:
+    """Given a list of scrobbles, update pages read, long play seconds and check
+    for media completion"""
+
+    for scrobble in scrobbles:
+        scrobble.book_pages_read = scrobble.book.page_set.last().number
+        # But if there's a next scrobble, set pages read to their starting page
+        #
+        if scrobble.next:
+            scrobble.book_pages_read = scrobble.next.book_pages_read - 1
+        scrobble.long_play_seconds = scrobble.book.page_set.filter(
+            number__lte=scrobble.book_pages_read
+        ).aggregate(Sum("duration_seconds"))["duration_seconds__sum"]
+
+        scrobble.save(update_fields=["book_pages_read", "long_play_seconds"])
+
+
+def process_koreader_sqlite_url(file_url, user_id) -> list:
+    book_id_map = {}
+    new_scrobbles = []
+
+    for table_name, pragma_table_info, rows in stream_sqlite(
+        _sqlite_bytes(file_url), max_buffer_size=1_048_576
+    ):
+        logger.debug(f"Found table {table_name} - processing")
+        if table_name == "book":
+            book_id_map = get_book_map_from_sqlite(rows)
+
+        if table_name == "page_stat_data":
+            new_scrobbles = build_scrobbles_from_pages(
+                rows, book_id_map, user_id
+            )
+            logger.debug(f"Creating {len(new_scrobbles)} new scrobbles")
+
+    created = []
+    if new_scrobbles:
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        enrich_koreader_scrobbles(created)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={"created_scrobbles": created},
+        )
+    return created
+
+
+def process_koreader_sqlite_file(file_path, user_id) -> list:
+    """Given a sqlite file from KoReader, open the book table, iterate
+    over rows creating scrobbles from each book found"""
+    # Create a SQL connection to our SQLite database
+    con = sqlite3.connect(file_path)
+    cur = con.cursor()
+
+    book_id_map = get_book_map_from_sqlite(cur.execute("SELECT * FROM book"))
+    new_scrobbles = build_scrobbles_from_pages(
+        cur.execute("SELECT * from page_stat_data"), book_id_map, user_id
+    )
+
+    created = []
+    if new_scrobbles:
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        enrich_koreader_scrobbles(created)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={"created_scrobbles": created},
+        )
+    return created
+
+
+def process_koreader_sqlite(file_path: str, user_id: int) -> list:
+    is_os_file = "https://" not in file_path
+
+    if is_os_file:
+        created = process_koreader_sqlite_file(file_path, user_id)
+    else:
+        created = process_koreader_sqlite_url(file_path, user_id)
+    return created

vrobbler/apps/books/migrations/0001_initial.py

@@ -0,0 +1,128 @@
+# Generated by Django 4.1.5 on 2023-02-19 20:17
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='Author',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Book',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'run_time',
+                    models.CharField(blank=True, max_length=8, null=True),
+                ),
+                (
+                    'run_time_ticks',
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                ('title', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'goodreads_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('koreader_id', models.IntegerField(blank=True, null=True)),
+                (
+                    'koreader_authors',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'koreader_md5',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'isbn',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('pages', models.IntegerField(blank=True, null=True)),
+                (
+                    'language',
+                    models.CharField(blank=True, max_length=4, null=True),
+                ),
+                (
+                    'first_publish_year',
+                    models.IntegerField(blank=True, null=True),
+                ),
+                ('authors', models.ManyToManyField(to='books.author')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/books/migrations/0002_book_author_name_book_author_openlibrary_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-06 05:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="author_name",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="author_openlibrary_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0003_book_cover.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-03-06 05:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0002_book_author_name_book_author_openlibrary_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="cover",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="books/covers/"
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0004_remove_book_author_name_and_more.py

@@ -0,0 +1,69 @@
+# Generated by Django 4.1.5 on 2023-03-06 16:31
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0003_book_cover"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="book",
+            name="author_name",
+        ),
+        migrations.RemoveField(
+            model_name="book",
+            name="author_openlibrary_id",
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="headshot",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="books/authors/"
+            ),
+        ),
+        migrations.CreateModel(
+            name="Page",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("number", models.IntegerField()),
+                ("start_time", models.DateTimeField()),
+                ("duration_seconds", models.IntegerField()),
+                (
+                    "book",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="books.book",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("book", "number")},
+            },
+        ),
+    ]

vrobbler/apps/books/migrations/0005_author_uuid.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.5 on 2023-03-06 16:34
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0004_remove_book_author_name_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="uuid",
+            field=models.UUIDField(
+                blank=True, default=uuid.uuid4, editable=False, null=True
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0006_alter_page_duration_seconds_alter_page_start_time.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-06 17:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0005_author_uuid"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="page",
+            name="duration_seconds",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="page",
+            name="start_time",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0007_author_amazon_id_author_bio_author_goodreads_id_and_more.py

@@ -0,0 +1,48 @@
+# Generated by Django 4.1.5 on 2023-03-06 17:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0006_alter_page_duration_seconds_alter_page_start_time"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="amazon_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="bio",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="goodreads_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="isni",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="librarything_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="wikidata_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="wikipedia_url",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0008_book_first_sentence.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.5 on 2023-03-06 18:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "books",
+            "0007_author_amazon_id_author_bio_author_goodreads_id_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="first_sentence",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0009_alter_book_run_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0008_book_first_sentence"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="run_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0010_rename_run_time_book_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0009_alter_book_run_time"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="book",
+            old_name="run_time",
+            new_name="run_time_seconds",
+        ),
+    ]

vrobbler/apps/books/migrations/0011_book_genre.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-14 22:27
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0033_genre_objectwithgenres"),
+        ("books", "0010_rename_run_time_book_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0012_page_end_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-26 02:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0011_book_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="page",
+            name="end_time",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0013_page_user.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-03-26 05:31
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("books", "0012_page_end_time"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="page",
+            name="user",
+            field=models.ForeignKey(
+                default=1,
+                on_delete=django.db.models.deletion.CASCADE,
+                to=settings.AUTH_USER_MODEL,
+            ),
+            preserve_default=False,
+        ),
+    ]

vrobbler/apps/books/models.py

@@ -0,0 +1,258 @@
+import logging
+from datetime import timedelta
+from uuid import uuid4
+
+import requests
+from books.openlibrary import (
+    lookup_author_from_openlibrary,
+    lookup_book_from_openlibrary,
+)
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.core.files.base import ContentFile
+from django.db import models
+from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
+from scrobbles.mixins import LongPlayScrobblableMixin, ScrobblableMixin
+from scrobbles.utils import get_scrobbles_for_media
+
+logger = logging.getLogger(__name__)
+User = get_user_model()
+BNULL = {"blank": True, "null": True}
+
+
+class Author(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+    headshot = models.ImageField(upload_to="books/authors/", **BNULL)
+    bio = models.TextField(**BNULL)
+    wikipedia_url = models.CharField(max_length=255, **BNULL)
+    isni = models.CharField(max_length=255, **BNULL)
+    wikidata_id = models.CharField(max_length=255, **BNULL)
+    goodreads_id = models.CharField(max_length=255, **BNULL)
+    librarything_id = models.CharField(max_length=255, **BNULL)
+    amazon_id = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return f"{self.name}"
+
+    def fix_metadata(self, data_dict: dict = {}):
+        if not data_dict and self.openlibrary_id:
+            data_dict = lookup_author_from_openlibrary(self.openlibrary_id)
+
+        if not data_dict or not data_dict.get("name"):
+            return
+
+        headshot_url = data_dict.pop("author_headshot_url", "")
+
+        Author.objects.filter(pk=self.id).update(**data_dict)
+        self.refresh_from_db()
+
+        if headshot_url:
+            r = requests.get(headshot_url)
+            if r.status_code == 200:
+                fname = f"{self.name}_{self.uuid}.jpg"
+                self.headshot.save(fname, ContentFile(r.content), save=True)
+
+
+class Book(LongPlayScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, "BOOK_COMPLETION_PERCENT", 95)
+    AVG_PAGE_READING_SECONDS = getattr(
+        settings, "AVERAGE_PAGE_READING_SECONDS", 60
+    )
+
+    title = models.CharField(max_length=255)
+    authors = models.ManyToManyField(Author)
+    goodreads_id = models.CharField(max_length=255, **BNULL)
+    koreader_id = models.IntegerField(**BNULL)
+    koreader_authors = models.CharField(max_length=255, **BNULL)
+    koreader_md5 = models.CharField(max_length=255, **BNULL)
+    isbn = models.CharField(max_length=255, **BNULL)
+    pages = models.IntegerField(**BNULL)
+    language = models.CharField(max_length=4, **BNULL)
+    first_publish_year = models.IntegerField(**BNULL)
+    first_sentence = models.CharField(max_length=255, **BNULL)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+    cover = models.ImageField(upload_to="books/covers/", **BNULL)
+
+    def __str__(self):
+        return f"{self.title} by {self.author}"
+
+    @property
+    def subtitle(self):
+        return f" by {self.author}"
+
+    @property
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.cover:
+            url = self.cover.url
+        return url
+
+    def get_start_url(self):
+        return reverse("scrobbles:start", kwargs={"uuid": self.uuid})
+
+    def get_absolute_url(self):
+        return reverse("books:book_detail", kwargs={"slug": self.uuid})
+
+    def fix_metadata(self, force_update=False):
+        if not self.openlibrary_id or force_update:
+            author_name = ""
+            if self.author:
+                author_name = self.author.name
+            book_dict = lookup_book_from_openlibrary(self.title, author_name)
+            if not book_dict:
+                logger.warn(f"Book not found in OL {self.title}")
+                return
+
+            cover_url = book_dict.pop("cover_url", "")
+            ol_author_id = book_dict.pop("ol_author_id", "")
+            ol_author_name = book_dict.pop("ol_author_name", "")
+            if book_dict.get("pages") == None:
+                book_dict.pop("pages")
+
+            ol_title = book_dict.get("title", "")
+
+            if ol_title.lower() != self.title.lower():
+                logger.warn(
+                    f"OL and KoReader disagree on this book title {self.title} != {ol_title}"
+                )
+
+            Book.objects.filter(pk=self.id).update(**book_dict)
+            self.refresh_from_db()
+
+            if cover_url:
+                r = requests.get(cover_url)
+                if r.status_code == 200:
+                    fname = f"{self.title}_{self.uuid}.jpg"
+                    self.cover.save(fname, ContentFile(r.content), save=True)
+
+            if self.pages:
+                self.run_time_seconds = self.pages * int(
+                    self.AVG_PAGE_READING_SECONDS
+                )
+
+            self.save()
+
+    @property
+    def author(self):
+        return self.authors.first()
+
+    @property
+    def pages_for_completion(self) -> int:
+        if not self.pages:
+            logger.warn(f"{self} has no pages, no completion percentage")
+            return 0
+        return int(self.pages * (self.COMPLETION_PERCENT / 100))
+
+    def update_long_play_seconds(self):
+        """Check page timestamps and duration and update"""
+        if self.page_set.all():
+            ...
+
+    def progress_for_user(self, user_id: int) -> int:
+        """Used to keep track of whether the book is complete or not"""
+        user = User.objects.get(id=user_id)
+        last_scrobble = get_scrobbles_for_media(self, user).last()
+        progress = 0
+        if last_scrobble:
+            progress = int((last_scrobble.book_pages_read / self.pages) * 100)
+        return progress
+
+    @classmethod
+    def find_or_create(cls, data_dict: dict) -> "Game":
+        from books.utils import update_or_create_book
+
+        return update_or_create_book(
+            data_dict.get("title"), data_dict.get("author")
+        )
+
+
+class Page(TimeStampedModel):
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    book = models.ForeignKey(Book, on_delete=models.CASCADE)
+    number = models.IntegerField()
+    start_time = models.DateTimeField(**BNULL)
+    end_time = models.DateTimeField(**BNULL)
+    duration_seconds = models.IntegerField(**BNULL)
+
+    class Meta:
+        unique_together = (
+            "book",
+            "number",
+        )
+
+    def __str__(self):
+        return f"Page {self.number} of {self.book.pages} in {self.book.title}"
+
+    def save(self, *args, **kwargs):
+        if not self.end_time and self.duration_seconds:
+            self._set_end_time()
+
+        return super(Page, self).save(*args, **kwargs)
+
+    @property
+    def next(self):
+        page = self.book.page_set.filter(number=self.number + 1).first()
+        if not page:
+            page = (
+                self.book.page_set.filter(created__gt=self.created)
+                .order_by("created")
+                .first()
+            )
+        return page
+
+    @property
+    def previous(self):
+        page = self.book.page_set.filter(number=self.number - 1).first()
+        if not page:
+            page = (
+                self.book.page_set.filter(created__lt=self.created)
+                .order_by("-created")
+                .first()
+            )
+        return page
+
+    @property
+    def seconds_to_next_page(self) -> int:
+        seconds = 999999  # Effectively infnity time as we have no next
+        if not self.end_time:
+            self._set_end_time()
+        if self.next:
+            seconds = (self.next.start_time - self.end_time).seconds
+        return seconds
+
+    @property
+    def is_scrobblable(self) -> bool:
+        """A page defines the start of a scrobble if the seconds to next page
+        are greater than an hour, or 3600 seconds, and it's not a single page,
+        so the next seconds to next_page is less than an hour as well.
+
+        As a special case, the first recorded page is a scrobble, so we establish
+        when the book was started.
+
+        """
+        is_scrobblable = False
+        over_an_hour_since_last_page = False
+        if not self.previous:
+            is_scrobblable = True
+
+        if self.previous:
+            over_an_hour_since_last_page = (
+                self.previous.seconds_to_next_page >= 3600
+            )
+        blip = self.seconds_to_next_page >= 3600
+
+        if over_an_hour_since_last_page and not blip:
+            is_scrobblable = True
+        return is_scrobblable
+
+    def _set_end_time(self) -> None:
+        if self.end_time:
+            return
+
+        self.end_time = self.start_time + timedelta(
+            seconds=self.duration_seconds
+        )
+        self.save(update_fields=["end_time"])

vrobbler/apps/books/openlibrary.py

@@ -0,0 +1,129 @@
+import json
+import logging
+import urllib
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+ISBN_URL = "https://openlibrary.org/isbn/{isbn}.json"
+SEARCH_URL = "https://openlibrary.org/search.json?q={query}&sort=editions&mode=everything"
+AUTHOR_SEARCH_URL = "https://openlibrary.org/search/authors.json?q={query}"
+COVER_URL = "https://covers.openlibrary.org/b/olid/{id}-L.jpg"
+AUTHOR_URL = "https://openlibrary.org/authors/{id}.json"
+AUTHOR_IMAGE_URL = "https://covers.openlibrary.org/a/olid/{id}-L.jpg"
+
+
+def get_first(key: str, result: dict) -> str:
+    obj = ""
+    if obj_list := result.get(key):
+        obj = obj_list[0]
+    return obj
+
+
+def get_author_openlibrary_id(name: str) -> str:
+    search_url = AUTHOR_SEARCH_URL.format(query=name)
+    response = requests.get(search_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return ""
+
+    results = json.loads(response.content)
+
+    if not results:
+        logger.warn(f"No author results found from search for {name}")
+        return ""
+
+    result = results.get("docs", [])
+    return result[0].get("key")
+
+
+def lookup_author_from_openlibrary(olid: str) -> dict:
+    author_url = AUTHOR_URL.format(id=olid)
+    response = requests.get(author_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return {}
+
+    results = json.loads(response.content)
+
+    if not results:
+        logger.warn(f"No author results found from OL for {olid}")
+        return {}
+
+    remote_ids = results.get("remote_ids", {})
+    bio = ""
+    if results.get("bio"):
+        try:
+            bio = results.get("bio").get("value")
+        except AttributeError:
+            bio = results.get("bio")
+    return {
+        "name": results.get("name"),
+        "openlibrary_id": olid,
+        "wikipedia_url": results.get("wikipedia"),
+        "wikidata_id": remote_ids.get("wikidata"),
+        "isni": remote_ids.get("isni"),
+        "goodreads_id": remote_ids.get("goodreads"),
+        "librarything_id": remote_ids.get("librarything"),
+        "amazon_id": remote_ids.get("amazon"),
+        "bio": bio,
+        "author_headshot_url": AUTHOR_IMAGE_URL.format(id=olid),
+    }
+
+
+def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
+    title_quoted = urllib.parse.quote(title)
+    author_quoted = ""
+    if author:
+        author_quoted = urllib.parse.quote(author)
+    query = f"{title_quoted} {author_quoted}"
+
+    search_url = SEARCH_URL.format(query=query)
+    response = requests.get(search_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return {}
+
+    results = json.loads(response.content)
+
+    if len(results.get("docs")) == 0:
+        logger.warn(f"No results found from OL for {title}")
+        return {}
+
+    top = None
+    for result in results.get("docs"):
+        # These Summary things suck and ruin our one-shot search
+        if "Summary of" not in result.get("title"):
+            top = result
+            break
+
+    if not top:
+        logger.warn(f"No book found for query {query}")
+        return {}
+
+    ol_id = top.get("cover_edition_key")
+    ol_author_id = get_first("author_key", top)
+    first_sentence = ""
+    if top.get("first_sentence"):
+        try:
+            first_sentence = top.get("first_sentence")[0].get("value")
+        except AttributeError:
+            first_sentence = top.get("first_sentence")[0]
+    isbn = None
+    if top.get("isbn"):
+        isbn = top.get("isbn")[0]
+    return {
+        "title": top.get("title"),
+        "isbn": isbn,
+        "openlibrary_id": ol_id,
+        "goodreads_id": get_first("id_goodreads", top),
+        "first_publish_year": top.get("first_publish_year"),
+        "first_sentence": first_sentence,
+        "pages": top.get("number_of_pages_median", None),
+        "cover_url": COVER_URL.format(id=ol_id),
+        "ol_author_id": ol_author_id,
+    }

vrobbler/apps/books/urls.py

@@ -0,0 +1,19 @@
+from django.urls import path
+from books import views
+
+app_name = "books"
+
+
+urlpatterns = [
+    path("book/", views.BookListView.as_view(), name="book_list"),
+    path(
+        "book/<slug:slug>/",
+        views.BookDetailView.as_view(),
+        name="book_detail",
+    ),
+    path(
+        "author/<slug:slug>/",
+        views.AuthorDetailView.as_view(),
+        name="author_detail",
+    ),
+]

vrobbler/apps/books/utils.py

@@ -0,0 +1,52 @@
+from django.core.files.base import ContentFile
+import requests
+from typing import Optional
+from books.openlibrary import (
+    lookup_author_from_openlibrary,
+    lookup_book_from_openlibrary,
+)
+from books.models import Author, Book
+
+
+def update_or_create_book(
+    title: str, author: Optional[str] = None, force_update=False
+) -> Book:
+    book_dict = lookup_book_from_openlibrary(title, author)
+
+    book, book_created = Book.objects.get_or_create(
+        isbn=book_dict.get("isbn"),
+    )
+
+    if book_created or force_update:
+        cover_url = book_dict.pop("cover_url")
+        ol_author_id = book_dict.pop("ol_author_id")
+
+        Book.objects.filter(pk=book.id).update(**book_dict)
+        book.refresh_from_db()
+
+        # Process authors
+        author_dict = lookup_author_from_openlibrary(ol_author_id)
+        author = Author.objects.filter(openlibrary_id=ol_author_id).first()
+        if not author:
+            author_image_url = author_dict.pop("author_headshot_url", None)
+
+            author = Author.objects.create(**author_dict)
+
+            if author_image_url:
+                r = requests.get(author_image_url)
+                if r.status_code == 200:
+                    fname = f"{author.name}_{author.uuid}.jpg"
+                    author.headshot.save(
+                        fname, ContentFile(r.content), save=True
+                    )
+        book.authors.add(author)
+
+        # Process cover URL
+        r = requests.get(cover_url)
+        if r.status_code == 200:
+            fname = f"{book.title}_{book.uuid}.jpg"
+            book.cover.save(fname, ContentFile(r.content), save=True)
+
+        book.fix_metadata()
+
+    return book

vrobbler/apps/books/views.py

@@ -0,0 +1,17 @@
+from django.views import generic
+from books.models import Book, Author
+
+
+class BookListView(generic.ListView):
+    model = Book
+    paginate_by = 20
+
+
+class BookDetailView(generic.DetailView):
+    model = Book
+    slug_field = "uuid"
+
+
+class AuthorDetailView(generic.DetailView):
+    model = Author
+    slug_field = "uuid"

vrobbler/apps/music/admin.py

@@ -2,20 +2,46 @@ from django.contrib import admin
 
 from music.models import Artist, Album, Track
 
+from scrobbles.admin import ScrobbleInline
+
 
 @admin.register(Album)
 class AlbumAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
-    list_display = ("name", "year", "musicbrainz_id")
-    list_filter = ("year",)
-    ordering = ("name",)
+    list_display = (
+        "name",
+        "year",
+        "album_artist",
+        "theaudiodb_genre",
+        "theaudiodb_mood",
+        "musicbrainz_id",
+    )
+    list_filter = (
+        "theaudiodb_score",
+        "theaudiodb_genre",
+    )
+    ordering = ("-created",)
+    search_fields = ("name",)
+    filter_horizontal = [
+        "artists",
+    ]
 
 
 @admin.register(Artist)
 class ArtistAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
-    list_display = ("name", "musicbrainz_id")
-    ordering = ("name",)
+    list_display = (
+        "name",
+        "theaudiodb_mood",
+        "theaudiodb_genre",
+        "musicbrainz_id",
+    )
+    list_filter = (
+        "theaudiodb_mood",
+        "theaudiodb_genre",
+    )
+    search_fields = ("name",)
+    ordering = ("-created",)
 
 
 @admin.register(Track)
@@ -25,8 +51,11 @@ class TrackAdmin(admin.ModelAdmin):
         "title",
         "album",
         "artist",
-        "run_time",
         "musicbrainz_id",
     )
     list_filter = ("album", "artist")
+    search_fields = ("title",)
     ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/music/aggregators.py

@@ -1,102 +1,151 @@
-from django.db.models import Q, Count, Sum
-from typing import List, Optional
-from scrobbles.models import Scrobble
-from music.models import Track, Artist
-from videos.models import Video
-
-from django.utils import timezone
 from datetime import datetime, timedelta
 
-
-NOW = timezone.now()
-START_OF_TODAY = datetime.combine(NOW.date(), datetime.min.time(), NOW.tzinfo)
-STARTING_DAY_OF_CURRENT_WEEK = NOW.date() - timedelta(
-    days=NOW.today().isoweekday() % 7
-)
-STARTING_DAY_OF_CURRENT_MONTH = NOW.date().replace(day=1)
-STARTING_DAY_OF_CURRENT_YEAR = NOW.date().replace(month=1, day=1)
+from django.apps import apps
+from django.db.models import Count, Q, QuerySet
+from django.utils import timezone
+from profiles.utils import now_user_timezone
+from scrobbles.models import Scrobble
+from videos.models import Video
 
 
-def scrobble_counts():
-    finished_scrobbles_qs = Scrobble.objects.filter(in_progress=False)
+def scrobble_counts(user=None):
+
+    now = timezone.now()
+    user_filter = Q()
+    if user and user.is_authenticated:
+        now = now_user_timezone(user.profile)
+        user_filter = Q(user=user)
+
+    start_of_today = datetime.combine(
+        now.date(), datetime.min.time(), now.tzinfo
+    )
+    starting_day_of_current_week = now.date() - timedelta(
+        days=now.today().isoweekday() % 7
+    )
+    starting_day_of_current_month = now.date().replace(day=1)
+    starting_day_of_current_year = now.date().replace(month=1, day=1)
+
+    finished_scrobbles_qs = Scrobble.objects.filter(
+        user_filter, played_to_completion=True
+    )
     data = {}
-    data['today'] = finished_scrobbles_qs.filter(
-        timestamp__gte=START_OF_TODAY
+    data["today"] = finished_scrobbles_qs.filter(
+        timestamp__gte=start_of_today
     ).count()
-    data['week'] = finished_scrobbles_qs.filter(
-        timestamp__gte=STARTING_DAY_OF_CURRENT_WEEK
+    data["week"] = finished_scrobbles_qs.filter(
+        timestamp__gte=starting_day_of_current_week
     ).count()
-    data['month'] = finished_scrobbles_qs.filter(
-        timestamp__gte=STARTING_DAY_OF_CURRENT_MONTH
+    data["month"] = finished_scrobbles_qs.filter(
+        timestamp__gte=starting_day_of_current_month
     ).count()
-    data['year'] = finished_scrobbles_qs.filter(
-        timestamp__gte=STARTING_DAY_OF_CURRENT_YEAR
+    data["year"] = finished_scrobbles_qs.filter(
+        timestamp__gte=starting_day_of_current_year
     ).count()
-    data['alltime'] = finished_scrobbles_qs.count()
+    data["alltime"] = finished_scrobbles_qs.count()
     return data
 
 
-def week_of_scrobbles(media: str = 'tracks') -> dict[str, int]:
+def week_of_scrobbles(
+    user=None, start=None, media: str = "tracks"
+) -> dict[str, int]:
+
+    now = timezone.now()
+    user_filter = Q()
+    if user and user.is_authenticated:
+        now = now_user_timezone(user.profile)
+        user_filter = Q(user=user)
+
+    if not start:
+        start = datetime.combine(now.date(), datetime.min.time(), now.tzinfo)
+
     scrobble_day_dict = {}
+    base_qs = Scrobble.objects.filter(user_filter, played_to_completion=True)
+
     media_filter = Q(track__isnull=False)
+    if media == "movies":
+        media_filter = Q(video__video_type=Video.VideoType.MOVIE)
+    if media == "series":
+        media_filter = Q(video__video_type=Video.VideoType.TV_EPISODE)
 
     for day in range(6, -1, -1):
-        start = START_OF_TODAY - timedelta(days=day)
-        end = datetime.combine(start, datetime.max.time(), NOW.tzinfo)
-        day_of_week = start.strftime('%A')
-        if media == 'movies':
-            media_filter = Q(video__videotype=Video.VideoType.MOVIE)
-        if media == 'series':
-            media_filter = Q(video__videotype=Video.VideoType.TV_EPISODE)
-        scrobble_day_dict[day_of_week] = (
-            Scrobble.objects.filter(media_filter)
-            .filter(
-                timestamp__gte=start,
-                timestamp__lte=end,
-                in_progress=False,
-            )
-            .count()
-        )
+        start_day = start - timedelta(days=day)
+        end = datetime.combine(start_day, datetime.max.time(), now.tzinfo)
+        day_of_week = start_day.strftime("%A")
+
+        scrobble_day_dict[day_of_week] = base_qs.filter(
+            media_filter,
+            timestamp__gte=start_day,
+            timestamp__lte=end,
+            played_to_completion=True,
+        ).count()
 
     return scrobble_day_dict
 
 
-def top_tracks(filter: str = "today", limit: int = 15) -> List["Track"]:
-    time_filter = Q(scrobble__timestamp__gte=START_OF_TODAY)
-    if filter == "week":
-        time_filter = Q(scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_WEEK)
-    if filter == "month":
-        time_filter = Q(scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_MONTH)
-    if filter == "year":
-        time_filter = Q(scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_YEAR)
+def live_charts(
+    user: "User",
+    media_type: str = "Track",
+    chart_period: str = "all",
+    limit: int = 15,
+) -> QuerySet:
+    now = timezone.now()
+    tzinfo = now.tzinfo
+    now = now.date()
+    if user.is_authenticated:
+        now = now_user_timezone(user.profile)
+        tzinfo = now.tzinfo
 
-    return (
-        Track.objects.annotate(num_scrobbles=Count("scrobble", distinct=True))
-        .filter(time_filter)
-        .order_by("-num_scrobbles")[:limit]
+    seven_days_ago = now - timedelta(days=7)
+    thirty_days_ago = now - timedelta(days=30)
+    start_of_today = datetime.combine(now, datetime.min.time(), tzinfo)
+    start_day_of_week = start_of_today - timedelta(
+        days=now.today().isoweekday() % 7
     )
+    start_day_of_month = now.replace(day=1)
+    start_day_of_year = now.replace(month=1, day=1)
 
+    media_model = apps.get_model(app_label="music", model_name=media_type)
 
-def top_artists(filter: str = "today", limit: int = 15) -> List["Artist"]:
-    time_filter = Q(track__scrobble__timestamp__gte=START_OF_TODAY)
-    if filter == "week":
-        time_filter = Q(
-            track__scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_WEEK
-        )
-    if filter == "month":
-        time_filter = Q(
-            track__scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_MONTH
-        )
-    if filter == "year":
-        time_filter = Q(
-            track__scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_YEAR
+    period_queries = {
+        "today": {"scrobble__timestamp__gte": start_of_today},
+        "week": {"scrobble__timestamp__gte": start_day_of_week},
+        "last7": {"scrobble__timestamp__gte": seven_days_ago},
+        "last30": {"scrobble__timestamp__gte": thirty_days_ago},
+        "month": {"scrobble__timestamp__gte": start_day_of_month},
+        "year": {"scrobble__timestamp__gte": start_day_of_year},
+        "all": {},
+    }
+
+    time_filter = Q()
+    completion_filter = Q(
+        scrobble__user=user, scrobble__played_to_completion=True
+    )
+    user_filter = Q(scrobble__user=user)
+    count_field = "scrobble"
+
+    if media_type == "Artist":
+        for period, query_dict in period_queries.items():
+            period_queries[period] = {
+                "track__" + k: v for k, v in query_dict.items()
+            }
+        completion_filter = Q(
+            track__scrobble__user=user,
+            track__scrobble__played_to_completion=True,
         )
+        count_field = "track__scrobble"
+        user_filter = Q(track__scrobble__user=user)
+
+    time_filter = Q(**period_queries[chart_period])
 
     return (
-        Artist.objects.annotate(
-            num_scrobbles=Count("track__scrobble", distinct=True)
+        media_model.objects.filter(user_filter, time_filter)
+        .annotate(
+            num_scrobbles=Count(
+                count_field,
+                filter=completion_filter,
+                distinct=True,
+            )
         )
-        .filter(time_filter)
         .order_by("-num_scrobbles")[:limit]
     )
 

vrobbler/apps/music/allmusic.py

@@ -0,0 +1,85 @@
+import urllib
+from typing import Optional
+from bs4 import BeautifulSoup
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+ALLMUSIC_SEARCH_URL = "https://www.allmusic.com/search/{subpath}/{query}"
+
+
+def strip_and_clean(text):
+    return text.strip("\n").rstrip().lstrip()
+
+
+def get_rating_from_soup(soup) -> Optional[int]:
+    rating = None
+    try:
+        potential_rating = soup.find("div", class_="allmusic-rating")
+        if potential_rating:
+            rating = int(strip_and_clean(potential_rating.get_text()))
+    except ValueError:
+        pass
+    return rating
+
+
+def get_review_from_soup(soup) -> str:
+    review = ""
+    try:
+        potential_text = soup.find("div", class_="text")
+        if potential_text:
+            review = strip_and_clean(potential_text.get_text())
+    except ValueError:
+        pass
+    return review
+
+
+def scrape_data_from_allmusic(url) -> dict:
+    data_dict = {}
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "html.parser")
+        data_dict["rating"] = get_rating_from_soup(soup)
+        data_dict["review"] = get_review_from_soup(soup)
+    return data_dict
+
+
+def get_allmusic_slug(artist_name=None, album_name=None) -> str:
+    slug = ""
+    if not artist_name:
+        return slug
+
+    subpath = "artists"
+    class_ = "name"
+    query = urllib.parse.quote(artist_name)
+    if album_name:
+        subpath = "albums"
+        class_ = "title"
+        query = "+".join([query, urllib.parse.quote(album_name)])
+
+    url = ALLMUSIC_SEARCH_URL.format(subpath=subpath, query=query)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+
+    if r.status_code != 200:
+        logger.info(f"Bad http response from Allmusic {r}")
+        return slug
+
+    soup = BeautifulSoup(r.text, "html.parser")
+    results = soup.find("ul", class_="search-results")
+
+    if not results:
+        logger.info(f"No search results for {query}")
+        return slug
+
+    prime_result = results.findAll("div", class_=class_)
+
+    if not prime_result:
+        logger.info(f"Could not find specific result for search {query}")
+
+    result_url = prime_result[0].find_all("a")[0]["href"]
+    slug = result_url.split("/")[-1:][0]
+
+    return slug

vrobbler/apps/music/api/serializers.py

@@ -0,0 +1,20 @@
+from music.models import Album, Artist, Track
+from rest_framework import serializers
+
+
+class ArtistSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Artist
+        fields = "__all__"
+
+
+class AlbumSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Album
+        fields = "__all__"
+
+
+class TrackSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Track
+        fields = "__all__"

vrobbler/apps/music/api/views.py

@@ -0,0 +1,26 @@
+from rest_framework import permissions, viewsets
+
+from music.api.serializers import (
+    TrackSerializer,
+    ArtistSerializer,
+    AlbumSerializer,
+)
+from music.models import Artist, Album, Track
+
+
+class ArtistViewSet(viewsets.ModelViewSet):
+    queryset = Artist.objects.all().order_by("-created")
+    serializer_class = ArtistSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class AlbumViewSet(viewsets.ModelViewSet):
+    queryset = Album.objects.all().order_by("-created")
+    serializer_class = AlbumSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class TrackViewSet(viewsets.ModelViewSet):
+    queryset = Track.objects.all().order_by("-created")
+    serializer_class = TrackSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/music/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class MusicConfig(AppConfig):
-    name = 'music'
+    name = "music"

vrobbler/apps/music/bandcamp.py

@@ -0,0 +1,49 @@
+import logging
+import urllib
+
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+BANDCAMP_SEARCH_URL = "https://bandcamp.com/search?q={query}&item_type={itype}"
+
+
+def get_bandcamp_slug(artist_name=None, album_name=None) -> str:
+    slug = ""
+    if not artist_name:
+        return slug
+
+    query = urllib.parse.quote(artist_name)
+    item_type = "b"
+    class_ = "heading"
+    if album_name:
+        item_type = "a"
+        query = "+".join([query, urllib.parse.quote(album_name)])
+
+    url = BANDCAMP_SEARCH_URL.format(query=query, itype=item_type)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+
+    if r.status_code != 200:
+        logger.info(f"Bad http response from Bandcamp {r}")
+        return slug
+
+    soup = BeautifulSoup(r.text, "html.parser")
+
+    results = soup.find("ul", class_="result-items")
+
+    if not results:
+        logger.info(f"No search results for {query}")
+        return slug
+
+    prime_result = results.findAll("div", class_=class_)
+
+    if not prime_result:
+        logger.info(f"Could not find specific result for search {query}")
+
+    result_url = prime_result[0].find_all("a")[0]["href"]
+    if item_type == "b":
+        slug = result_url.split("/")[2].split(".")[0]
+    else:
+        slug = result_url.split("?")[0]
+    return slug

vrobbler/apps/music/constants.py

@@ -1,16 +1,21 @@
-JELLYFIN_POST_KEYS = {
-    'ITEM_TYPE': 'ItemType',
-    'RUN_TIME_TICKS': 'RunTimeTicks',
-    'RUN_TIME': 'RunTime',
-    'TITLE': 'Name',
-    'TIMESTAMP': 'UtcTimestamp',
-    'YEAR': 'Year',
-    'PLAYBACK_POSITION_TICKS': 'PlaybackPositionTicks',
-    'PLAYBACK_POSITION': 'PlaybackPosition',
-    'ARTIST_MB_ID': 'Provider_musicbrainzartist',
-    'ALBUM_MB_ID': 'Provider_musicbrainzalbum',
-    'RELEASEGROUP_MB_ID': 'Provider_musicbrainzreleasegroup',
-    'TRACK_MB_ID': 'Provider_musicbrainztrack',
-    'ALBUM_NAME': 'Album',
-    'ARTIST_NAME': 'Artist',
+VARIOUS_ARTIST_DICT = {
+    "name": "Various Artists",
+    "theaudiodb_id": "113641",
+    "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377",
+}
+
+JELLYFIN_POST_KEYS = {
+    "ITEM_TYPE": "ItemType",
+    "RUN_TIME": "RunTime",
+    "TITLE": "Name",
+    "TIMESTAMP": "UtcTimestamp",
+    "YEAR": "Year",
+    "PLAYBACK_POSITION_TICKS": "PlaybackPositionTicks",
+    "PLAYBACK_POSITION": "PlaybackPosition",
+    "ARTIST_MB_ID": "Provider_musicbrainzartist",
+    "ALBUM_MB_ID": "Provider_musicbrainzalbum",
+    "RELEASEGROUP_MB_ID": "Provider_musicbrainzreleasegroup",
+    "TRACK_MB_ID": "Provider_musicbrainztrack",
+    "ALBUM_NAME": "Album",
+    "ARTIST_NAME": "Artist",
 }

vrobbler/apps/music/lastfm.py

@@ -0,0 +1,144 @@
+import logging
+import time
+from datetime import datetime, timedelta
+
+import pylast
+import pytz
+from django.conf import settings
+from django.utils import timezone
+from music.utils import (
+    get_or_create_album,
+    get_or_create_artist,
+    get_or_create_track,
+)
+
+logger = logging.getLogger(__name__)
+
+PYLAST_ERRORS = tuple(
+    getattr(pylast, exc_name)
+    for exc_name in (
+        "ScrobblingError",
+        "NetworkError",
+        "MalformedResponseError",
+        "WSError",
+    )
+    if hasattr(pylast, exc_name)
+)
+
+
+class LastFM:
+    def __init__(self, user):
+        try:
+            self.client = pylast.LastFMNetwork(
+                api_key=getattr(settings, "LASTFM_API_KEY"),
+                api_secret=getattr(settings, "LASTFM_SECRET_KEY"),
+                username=user.profile.lastfm_username,
+                password_hash=pylast.md5(user.profile.lastfm_password),
+            )
+            self.user = self.client.get_user(user.profile.lastfm_username)
+            self.vrobbler_user = user
+        except PYLAST_ERRORS as e:
+            logger.error(f"Error during Last.fm setup: {e}")
+
+    def import_from_lastfm(self, last_processed=None):
+        """Given a last processed time, import all scrobbles from LastFM since then"""
+        from scrobbles.models import Scrobble
+
+        new_scrobbles = []
+        source = "Last.fm"
+        source_id = ""
+        lastfm_scrobbles = self.get_last_scrobbles(time_from=last_processed)
+
+        for lfm_scrobble in lastfm_scrobbles:
+            timestamp = lfm_scrobble.pop("timestamp")
+
+            artist = get_or_create_artist(lfm_scrobble.pop("artist"))
+            album = get_or_create_album(lfm_scrobble.pop("album"), artist)
+
+            lfm_scrobble["artist"] = artist
+            if album:
+                lfm_scrobble["album"] = album
+            track = get_or_create_track(**lfm_scrobble)
+
+            new_scrobble = Scrobble(
+                user=self.vrobbler_user,
+                timestamp=timestamp,
+                source=source,
+                source_id=source_id,
+                track=track,
+                played_to_completion=True,
+                in_progress=False,
+            )
+            # Vrobbler scrobbles on finish, LastFM scrobbles on start
+            seconds_eariler = timestamp - timedelta(seconds=20)
+            seconds_later = timestamp + timedelta(seconds=20)
+            existing = Scrobble.objects.filter(
+                created__gte=seconds_eariler,
+                created__lte=seconds_later,
+                track=track,
+            ).first()
+            if existing:
+                logger.debug(f"Skipping existing scrobble {new_scrobble}")
+                continue
+            logger.debug(f"Queued scrobble {new_scrobble} for creation")
+            new_scrobbles.append(new_scrobble)
+
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={"created_scrobbles": created},
+        )
+        return created
+
+    def get_last_scrobbles(self, time_from=None, time_to=None):
+        """Given a user, Last.fm api key, and secret key, grab a list of scrobbled
+        tracks"""
+        lfm_params = {}
+        scrobbles = []
+        if time_from:
+            lfm_params["time_from"] = int(time_from.timestamp())
+        if time_to:
+            lfm_params["time_to"] = int(time_to.timestamp())
+
+        # if not time_from and not time_to:
+        lfm_params["limit"] = None
+
+        found_scrobbles = self.user.get_recent_tracks(**lfm_params)
+        # TOOD spin this out into a celery task over certain threshold of found scrobbles?
+
+        for scrobble in found_scrobbles:
+            run_time = None
+            mbid = None
+            artist = None
+
+            try:
+                run_time = int(scrobble.track.get_duration() / 1000)
+                mbid = scrobble.track.get_mbid()
+                artist = scrobble.track.get_artist().name
+            except pylast.MalformedResponseError as e:
+                logger.warn(e)
+            except pylast.WSError as e:
+                logger.warn(
+                    "LastFM barfed trying to get the track for {scrobble.track}"
+                )
+
+            if not artist:
+                logger.warn(f"Silly LastFM, no artist found for {scrobble}")
+                continue
+
+            timestamp = datetime.utcfromtimestamp(
+                int(scrobble.timestamp)
+            ).replace(tzinfo=pytz.utc)
+
+            logger.info(f"{artist},{scrobble.track.title},{timestamp}")
+            scrobbles.append(
+                {
+                    "artist": artist,
+                    "album": scrobble.album,
+                    "title": scrobble.track.title,
+                    "mbid": mbid,
+                    "run_time_seconds": run_time,
+                    "timestamp": timestamp,
+                }
+            )
+        return scrobbles

vrobbler/apps/music/migrations/0005_album_cover_image.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-01-12 04:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            'music',
+            '0004_alter_artist_options_alter_album_musicbrainz_id_and_more',
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='cover_image',
+            field=models.ImageField(
+                blank=True, null=True, upload_to='albums/'
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0006_album_artists.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-01-12 05:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0005_album_cover_image'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='artists',
+            field=models.ManyToManyField(
+                blank=True, null=True, to='music.artist'
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0007_alter_album_artists.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-01-12 17:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0006_album_artists'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='album',
+            name='artists',
+            field=models.ManyToManyField(to='music.artist'),
+        ),
+    ]

vrobbler/apps/music/migrations/0008_alter_track_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.5 on 2023-01-13 01:47
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0007_alter_album_artists'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='track',
+            options={},
+        ),
+    ]

vrobbler/apps/music/migrations/0009_alter_track_musicbrainz_id_and_more.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.5 on 2023-01-19 20:03
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0008_alter_track_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='track',
+            name='musicbrainz_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AlterUniqueTogether(
+            name='track',
+            unique_together={('album', 'musicbrainz_id')},
+        ),
+    ]

vrobbler/apps/music/migrations/0010_artist_biography_artist_theaudiodb_genre_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.1.5 on 2023-02-27 03:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0009_alter_track_musicbrainz_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='artist',
+            name='biography',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='artist',
+            name='theaudiodb_genre',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='artist',
+            name='theaudiodb_mood',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0011_artist_thumbnail.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-02-27 04:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0010_artist_biography_artist_theaudiodb_genre_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='artist',
+            name='thumbnail',
+            field=models.ImageField(
+                blank=True, null=True, upload_to='artist/'
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0012_album_allmusic_id_album_discogs_id_and_more.py

@@ -0,0 +1,85 @@
+# Generated by Django 4.1.5 on 2023-03-02 19:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0011_artist_thumbnail'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='allmusic_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='discogs_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='rateyourmusic_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_description',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_genre',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_id',
+            field=models.CharField(
+                blank=True, max_length=255, null=True, unique=True
+            ),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_mood',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_score',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_score_votes',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_speed',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_style',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_theme',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='wikidata_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='wikipedia_slug',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0013_alter_album_theaudiodb_score.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-02 19:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0012_album_allmusic_id_album_discogs_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='album',
+            name='theaudiodb_score',
+            field=models.FloatField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0014_album_theaudiodb_year_released.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-02 19:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0013_alter_album_theaudiodb_score'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_year_released',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0015_alter_track_run_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0014_album_theaudiodb_year_released"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="track",
+            name="run_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0016_rename_run_time_track_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0015_alter_track_run_time"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="track",
+            old_name="run_time",
+            new_name="run_time_seconds",
+        ),
+    ]

vrobbler/apps/music/migrations/0017_track_genre.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-14 22:27
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0033_genre_objectwithgenres"),
+        ("music", "0016_rename_run_time_track_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="track",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0018_album_allmusic_rating_album_allmusic_review.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-15 01:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0017_track_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="allmusic_rating",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="album",
+            name="allmusic_review",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0019_artist_allmusic_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-15 03:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0018_album_allmusic_rating_album_allmusic_review"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="artist",
+            name="allmusic_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0020_album_bandcamp_id_artist_bandcamp_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-15 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0019_artist_allmusic_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="bandcamp_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name="artist",
+            name="bandcamp_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0021_album_album_artist.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-15 16:22
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0020_album_bandcamp_id_artist_bandcamp_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="album_artist",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                related_name="albums",
+                to="music.artist",
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0022_artist_theaudiodb_id.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-03-15 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0021_album_album_artist"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="artist",
+            name="theaudiodb_id",
+            field=models.CharField(
+                blank=True, max_length=255, null=True, unique=True
+            ),
+        ),
+    ]

vrobbler/apps/music/models.py

@@ -1,40 +1,40 @@
 import logging
+from tempfile import NamedTemporaryFile
 from typing import Dict, Optional
+from urllib.request import urlopen
 from uuid import uuid4
 
-from django.apps.config import cached_property
-
+import musicbrainzngs
+import requests
+from django.conf import settings
+from django.core.files.base import ContentFile, File
 from django.db import models
+from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
+from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic
+from music.bandcamp import get_bandcamp_slug
+from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb
+from scrobbles.mixins import ScrobblableMixin
 
 logger = logging.getLogger(__name__)
 BNULL = {"blank": True, "null": True}
 
 
-class Album(TimeStampedModel):
-    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
-    name = models.CharField(max_length=255)
-    year = models.IntegerField(**BNULL)
-    musicbrainz_id = models.CharField(max_length=255, unique=True, **BNULL)
-    musicbrainz_releasegroup_id = models.CharField(max_length=255, **BNULL)
-    musicbrainz_albumartist_id = models.CharField(max_length=255, **BNULL)
-
-    def __str__(self):
-        return self.name
-
-    @property
-    def mb_link(self):
-        return f"https://musicbrainz.org/release/{self.musicbrainz_id}"
-
-
 class Artist(TimeStampedModel):
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     name = models.CharField(max_length=255)
+    biography = models.TextField(**BNULL)
+    theaudiodb_id = models.CharField(max_length=255, unique=True, **BNULL)
+    theaudiodb_genre = models.CharField(max_length=255, **BNULL)
+    theaudiodb_mood = models.CharField(max_length=255, **BNULL)
     musicbrainz_id = models.CharField(max_length=255, **BNULL)
+    allmusic_id = models.CharField(max_length=100, **BNULL)
+    bandcamp_id = models.CharField(max_length=100, **BNULL)
+    thumbnail = models.ImageField(upload_to="artist/", **BNULL)
 
     class Meta:
-        unique_together=[['name', 'musicbrainz_id']]
+        unique_together = [["name", "musicbrainz_id"]]
 
     def __str__(self):
         return self.name
@@ -43,32 +43,359 @@ class Artist(TimeStampedModel):
     def mb_link(self):
         return f"https://musicbrainz.org/artist/{self.musicbrainz_id}"
 
+    @property
+    def allmusic_link(self):
+        if self.allmusic_id:
+            return f"https://www.allmusic.com/artist/{self.allmusic_id}"
+        return ""
 
-class Track(TimeStampedModel):
-    class Opinion(models.IntegerChoices):
-        DOWN = -1, 'Thumbs down'
-        NEUTRAL = 0, 'No opinion'
-        UP = 1, 'Thumbs up'
+    @property
+    def bandcamp_link(self):
+        if self.bandcamp_id:
+            return f"https://{self.bandcamp_id}.bandcamp.com/"
+        return ""
 
+    def get_absolute_url(self):
+        return reverse("music:artist_detail", kwargs={"slug": self.uuid})
+
+    def scrobbles(self):
+        from scrobbles.models import Scrobble
+
+        return Scrobble.objects.filter(
+            track__in=self.track_set.all()
+        ).order_by("-timestamp")
+
+    @property
+    def tracks(self):
+        return (
+            self.track_set.all()
+            .annotate(scrobble_count=models.Count("scrobble"))
+            .order_by("-scrobble_count")
+        )
+
+    def charts(self):
+        from scrobbles.models import ChartRecord
+
+        return ChartRecord.objects.filter(track__artist=self).order_by("-year")
+
+    def scrape_allmusic(self, force=False) -> None:
+        if not self.allmusic_id or force:
+            slug = get_allmusic_slug(self.name)
+            if not slug:
+                logger.info(f"No allmsuic link for {self}")
+                return
+            self.allmusic_id = slug
+            self.save(update_fields=["allmusic_id"])
+
+    def scrape_bandcamp(self, force=False) -> None:
+        if not self.bandcamp_id or force:
+            slug = get_bandcamp_slug(self.name)
+            if not slug:
+                logger.info(f"No bandcamp link for {self}")
+                return
+            self.bandcamp_id = slug
+            self.save(update_fields=["bandcamp_id"])
+
+    def fix_metadata(self):
+        tadb_info = lookup_artist_from_tadb(self.name)
+        if not tadb_info:
+            logger.warn(f"No response from TADB for artist {self.name}")
+            return
+
+        self.biography = tadb_info["biography"]
+        self.theaudiodb_genre = tadb_info["genre"]
+        self.theaudiodb_mood = tadb_info["mood"]
+
+        thumb_url = tadb_info.get("thumb_url", "")
+        if thumb_url:
+            r = requests.get(thumb_url)
+            if r.status_code == 200:
+                fname = f"{self.name}_{self.uuid}.jpg"
+                self.thumbnail.save(fname, ContentFile(r.content), save=True)
+
+    @property
+    def rym_link(self):
+        artist_slug = self.name.lower().replace(" ", "-")
+        return f"https://rateyourmusic.com/artist/{artist_slug}/"
+
+    @property
+    def bandcamp_search_link(self):
+        artist = self.name.lower()
+        return f"https://bandcamp.com/search?q={artist}&item_type=b"
+
+
+class Album(TimeStampedModel):
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
-    title = models.CharField(max_length=255, **BNULL)
+    name = models.CharField(max_length=255)
+    album_artist = models.ForeignKey(
+        Artist, related_name="albums", on_delete=models.DO_NOTHING, **BNULL
+    )
+    artists = models.ManyToManyField(Artist)
+    year = models.IntegerField(**BNULL)
+    musicbrainz_id = models.CharField(max_length=255, unique=True, **BNULL)
+    musicbrainz_releasegroup_id = models.CharField(max_length=255, **BNULL)
+    musicbrainz_albumartist_id = models.CharField(max_length=255, **BNULL)
+    cover_image = models.ImageField(upload_to="albums/", **BNULL)
+    theaudiodb_id = models.CharField(max_length=255, unique=True, **BNULL)
+    theaudiodb_description = models.TextField(**BNULL)
+    theaudiodb_year_released = models.IntegerField(**BNULL)
+    theaudiodb_score = models.FloatField(**BNULL)
+    theaudiodb_score_votes = models.IntegerField(**BNULL)
+    theaudiodb_genre = models.CharField(max_length=255, **BNULL)
+    theaudiodb_style = models.CharField(max_length=255, **BNULL)
+    theaudiodb_mood = models.CharField(max_length=255, **BNULL)
+    theaudiodb_speed = models.CharField(max_length=255, **BNULL)
+    theaudiodb_theme = models.CharField(max_length=255, **BNULL)
+    allmusic_id = models.CharField(max_length=255, **BNULL)
+    allmusic_rating = models.IntegerField(**BNULL)
+    allmusic_review = models.TextField(**BNULL)
+    bandcamp_id = models.CharField(max_length=100, **BNULL)
+    rateyourmusic_id = models.CharField(max_length=255, **BNULL)
+    wikipedia_slug = models.CharField(max_length=255, **BNULL)
+    discogs_id = models.CharField(max_length=255, **BNULL)
+    wikidata_id = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse("music:album_detail", kwargs={"slug": self.uuid})
+
+    def scrobbles(self):
+        from scrobbles.models import Scrobble
+
+        return Scrobble.objects.filter(
+            track__in=self.track_set.all()
+        ).order_by("-timestamp")
+
+    @property
+    def tracks(self):
+        return (
+            self.track_set.all()
+            .annotate(scrobble_count=models.Count("scrobble"))
+            .order_by("-scrobble_count")
+        )
+
+    def fix_album_artist(self):
+        from music.utils import get_or_create_various_artists
+
+        multiple_artists = self.artists.count() > 1
+        if multiple_artists:
+            self.album_artist = get_or_create_various_artists()
+        else:
+            self.album_artist = self.artists.first()
+        self.save(update_fields=["album_artist"])
+
+    def scrape_allmusic(self, force=False) -> None:
+        if not self.allmusic_id or force:
+            slug = get_allmusic_slug(self.name, self.album_artist.name)
+            if not slug:
+                logger.info(
+                    f"No allmsuic link for {self} by {self.album_artist}"
+                )
+                return
+            self.allmusic_id = slug
+            self.save(update_fields=["allmusic_id"])
+
+        allmusic_data = scrape_data_from_allmusic(self.allmusic_link)
+
+        if not allmusic_data:
+            logger.info(f"No allmsuic data for {self} by {self.album_artist}")
+            return
+
+        self.allmusic_review = allmusic_data["review"]
+        self.allmusic_rating = allmusic_data["rating"]
+        self.save(update_fields=["allmusic_review", "allmusic_rating"])
+
+    def scrape_theaudiodb(self) -> None:
+        artist = "Various Artists"
+        if self.album_artist:
+            artist = self.album_artist.name
+        album_data = lookup_album_from_tadb(self.name, artist)
+        if not album_data.get("theaudiodb_id"):
+            logger.info(f"No data for {self} found in TheAudioDB")
+            return
+
+        Album.objects.filter(pk=self.pk).update(**album_data)
+
+    def scrape_bandcamp(self, force=False) -> None:
+        if not self.bandcamp_id or force:
+            slug = get_bandcamp_slug(self.album_artist.name, self.name)
+            if not slug:
+                logger.info(f"No bandcamp link for {self}")
+                return
+            self.bandcamp_id = slug
+            self.save(update_fields=["bandcamp_id"])
+
+    def fix_metadata(self):
+        if (
+            not self.musicbrainz_albumartist_id
+            or not self.year
+            or not self.musicbrainz_releasegroup_id
+        ):
+            musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+            mb_data = musicbrainzngs.get_release_by_id(
+                self.musicbrainz_id, includes=["artists", "release-groups"]
+            )
+            if not self.musicbrainz_releasegroup_id:
+                self.musicbrainz_releasegroup_id = mb_data["release"][
+                    "release-group"
+                ]["id"]
+            if not self.musicbrainz_albumartist_id:
+                self.musicbrainz_albumartist_id = mb_data["release"][
+                    "artist-credit"
+                ][0]["artist"]["id"]
+            if not self.year:
+                try:
+                    self.year = mb_data["release"]["date"][0:4]
+                except KeyError:
+                    pass
+                except IndexError:
+                    pass
+
+            self.save(
+                update_fields=[
+                    "musicbrainz_albumartist_id",
+                    "musicbrainz_releasegroup_id",
+                    "year",
+                ]
+            )
+
+            new_artist = Artist.objects.filter(
+                musicbrainz_id=self.musicbrainz_albumartist_id
+            ).first()
+            if self.musicbrainz_albumartist_id and new_artist:
+                self.artists.add(new_artist)
+            if not new_artist:
+                for t in self.track_set.all():
+                    self.artists.add(t.artist)
+            if (
+                not self.cover_image
+                or self.cover_image == "default-image-replace-me"
+            ):
+                self.fetch_artwork()
+        self.fix_album_artist()
+        self.scrape_theaudiodb()
+        self.scrape_allmusic()
+
+    def fetch_artwork(self, force=False):
+        if not self.cover_image and not force:
+            if self.musicbrainz_id:
+                try:
+                    img_data = musicbrainzngs.get_image_front(
+                        self.musicbrainz_id
+                    )
+                    name = f"{self.name}_{self.uuid}.jpg"
+                    self.cover_image = ContentFile(img_data, name=name)
+                    logger.info(f"Setting image to {name}")
+                except musicbrainzngs.ResponseError:
+                    logger.warning(
+                        f"No cover art found for {self.name} by release"
+                    )
+
+            if (
+                not self.cover_image
+                or self.cover_image == "default-image-replace-me"
+            ) and self.musicbrainz_releasegroup_id:
+                try:
+                    img_data = musicbrainzngs.get_release_group_image_front(
+                        self.musicbrainz_releasegroup_id
+                    )
+                    name = f"{self.name}_{self.uuid}.jpg"
+                    self.cover_image = ContentFile(img_data, name=name)
+                    logger.info(f"Setting image to {name}")
+                except musicbrainzngs.ResponseError:
+                    logger.warning(
+                        f"No cover art found for {self.name} by release group"
+                    )
+            if not self.cover_image:
+                logger.debug(
+                    f"No cover art found for release or release group for {self.name}, setting to default"
+                )
+            self.save()
+
+    @property
+    def mb_link(self) -> str:
+        return f"https://musicbrainz.org/release/{self.musicbrainz_id}"
+
+    @property
+    def allmusic_link(self) -> str:
+        if self.allmusic_id:
+            return f"https://www.allmusic.com/album/{self.allmusic_id}"
+        return ""
+
+    @property
+    def wikipedia_link(self):
+        if self.wikipedia_slug:
+            return f"https://www.wikipedia.org/en/{self.wikipedia_slug}"
+        return ""
+
+    @property
+    def tadb_link(self):
+        if self.theaudiodb_id:
+            return f"https://www.theaudiodb.com/album/{self.theaudiodb_id}"
+        return ""
+
+    @property
+    def rym_link(self):
+        artist_slug = self.album_artist.name.lower().replace(" ", "-")
+        album_slug = self.name.lower().replace(" ", "-")
+        return f"https://rateyourmusic.com/release/album/{artist_slug}/{album_slug}/"
+
+    @property
+    def bandcamp_link(self):
+        if self.bandcamp_id and self.album_artist.bandcamp_id:
+            return f"https://{self.album_artist.bandcamp_id}.bandcamp.com/album/{self.bandcamp_id}"
+        return ""
+
+    @property
+    def bandcamp_search_link(self):
+        artist = self.album_artist.name.lower()
+        album = self.name.lower()
+        return f"https://bandcamp.com/search?q={album} {artist}&item_type=a"
+
+
+class Track(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, "MUSIC_COMPLETION_PERCENT", 90)
+
+    class Opinion(models.IntegerChoices):
+        DOWN = -1, "Thumbs down"
+        NEUTRAL = 0, "No opinion"
+        UP = 1, "Thumbs up"
+
     artist = models.ForeignKey(Artist, on_delete=models.DO_NOTHING)
     album = models.ForeignKey(Album, on_delete=models.DO_NOTHING, **BNULL)
-    musicbrainz_id = models.CharField(max_length=255, unique=True, **BNULL)
-    run_time = models.CharField(max_length=8, **BNULL)
-    run_time_ticks = models.PositiveBigIntegerField(**BNULL)
-    # thumbs = models.IntegerField(default=Opinion.NEUTRAL, choices=Opinion.choices)
+    musicbrainz_id = models.CharField(max_length=255, **BNULL)
+
+    class Meta:
+        unique_together = [["album", "musicbrainz_id"]]
 
     def __str__(self):
         return f"{self.title} by {self.artist}"
 
+    def get_absolute_url(self):
+        return reverse("music:track_detail", kwargs={"slug": self.uuid})
+
+    @property
+    def subtitle(self):
+        return self.artist
+
     @property
     def mb_link(self):
         return f"https://musicbrainz.org/recording/{self.musicbrainz_id}"
 
-    @cached_property
-    def scrobble_count(self):
-        return self.scrobble_set.filter(in_progress=False).count()
+    @property
+    def info_link(self):
+        return self.mb_link
+
+    @property
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.artist.thumbnail:
+            url = self.artist.thumbnail.url
+        if self.album and self.album.cover_image:
+            url = self.album.cover_image.url
+        return url
 
     @classmethod
     def find_or_create(
@@ -79,32 +406,24 @@ class Track(TimeStampedModel):
         exist.
 
         """
-        if not artist_dict.get('name') or not artist_dict.get(
-            'musicbrainz_id'
+        if not artist_dict.get("name") or not artist_dict.get(
+            "musicbrainz_id"
         ):
             logger.warning(
-                f"No artist or artist musicbrainz ID found in message from Jellyfin, not scrobbling"
+                f"No artist or artist musicbrainz ID found in message from source, not scrobbling"
             )
             return
+
         artist, artist_created = Artist.objects.get_or_create(**artist_dict)
-        if artist_created:
-            logger.debug(f"Created new album {artist}")
-        else:
-            logger.debug(f"Found album {artist}")
-
         album, album_created = Album.objects.get_or_create(**album_dict)
-        if album_created:
-            logger.debug(f"Created new album {album}")
-        else:
-            logger.debug(f"Found album {album}")
 
-        track_dict['album_id'] = getattr(album, "id", None)
-        track_dict['artist_id'] = artist.id
+        album.fix_metadata()
+        if not album.cover_image:
+            album.fetch_artwork()
+
+        track_dict["album_id"] = getattr(album, "id", None)
+        track_dict["artist_id"] = artist.id
 
         track, created = cls.objects.get_or_create(**track_dict)
-        if created:
-            logger.debug(f"Created new track: {track}")
-        else:
-            logger.debug(f"Found track {track}")
 
         return track

vrobbler/apps/music/musicbrainz.py

@@ -0,0 +1,110 @@
+import logging
+
+import musicbrainzngs
+from dateutil.parser import parse
+
+logger = logging.getLogger(__name__)
+
+
+def lookup_album_from_mb(musicbrainz_id: str) -> dict:
+    release_dict = {}
+
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+    release_data = musicbrainzngs.get_release_by_id(
+        musicbrainz_id,
+        includes=["artists", "release-groups", "recordings"],
+    ).get("release")
+
+    if not release_data:
+        return release_dict
+
+    primary_artist = release_data.get("artist-credit")[0]
+    release_dict = {
+        "artist": {
+            "name": primary_artist.get("name"),
+            "musicbrainz_id": primary_artist.get("id"),
+        },
+        "album": {
+            "name": release_data.get("title"),
+            "musicbrainz_id": musicbrainz_id,
+            "musicbrainz_releasegroup_id": release_data.get(
+                "release-group"
+            ).get("id"),
+            "musicbrainz_albumaritist_id": primary_artist.get("id"),
+            "year": release_data.get("year")[0:4],
+        },
+    }
+
+    release_dict["tracks"] = []
+    for track in release_data.get("medium-list")[0]["track-list"]:
+        recording = track["recording"]
+        release_dict["tracks"].append(
+            {
+                "title": recording["title"],
+                "musicbrainz_id": recording["id"],
+                "run_time": track["length"] / 1000,
+            }
+        )
+
+    return release_dict
+
+
+def lookup_album_dict_from_mb(release_name: str, artist_name: str) -> dict:
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+
+    top_result = musicbrainzngs.search_releases(
+        release_name, artist=artist_name
+    )["release-list"][0]
+    score = int(top_result.get("ext:score"))
+    if score < 85:
+        logger.debug(
+            "Album lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return {}
+    year = None
+    if top_result.get("date"):
+        year = parse(top_result["date"]).year
+
+    return {
+        "year": year,
+        "title": top_result["title"],
+        "mb_id": top_result["id"],
+        "mb_group_id": top_result["release-group"]["id"],
+    }
+
+
+def lookup_artist_from_mb(artist_name: str) -> str:
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+
+    top_result = musicbrainzngs.search_artists(artist=artist_name)[
+        "artist-list"
+    ][0]
+    score = int(top_result.get("ext:score"))
+    if score < 85:
+        logger.debug(
+            "Artist lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return ""
+
+    return top_result
+
+
+def lookup_track_from_mb(
+    track_name: str, artist_mbid: str, album_mbid: str
+) -> str:
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+
+    top_result = musicbrainzngs.search_recordings(
+        query=track_name, artist=artist_mbid, release=album_mbid
+    )["recording-list"][0]
+    score = int(top_result.get("ext:score"))
+    if score < 85:
+        logger.debug(
+            "Track lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return ""
+
+    return top_result

vrobbler/apps/music/serializers.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

vrobbler/apps/music/theaudiodb.py

@@ -0,0 +1,82 @@
+import urllib
+import json
+import logging
+
+import requests
+from django.conf import settings
+
+THEAUDIODB_API_KEY = getattr(settings, "THEAUDIODB_API_KEY")
+ARTIST_SEARCH_URL = f"https://www.theaudiodb.com/api/v1/json/{THEAUDIODB_API_KEY}/search.php?s="
+ALBUM_SEARCH_URL = f"https://www.theaudiodb.com/api/v1/json/{THEAUDIODB_API_KEY}/searchalbum.php?s="
+
+logger = logging.getLogger(__name__)
+
+
+def lookup_artist_from_tadb(name: str) -> dict:
+    artist_info = {}
+    name = urllib.parse.quote(name)
+    response = requests.get(ARTIST_SEARCH_URL + name)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from TADB: {response.status_code}")
+        return {}
+
+    if not response.content:
+        logger.warn(f"Bad content from TADB: {response.content}")
+        return {}
+
+    results = json.loads(response.content)
+    if results["artists"]:
+        artist = results["artists"][0]
+
+        artist_info["biography"] = artist.get("strBiographyEN")
+        artist_info["genre"] = artist.get("strGenre")
+        artist_info["mood"] = artist.get("strMood")
+        artist_info["thumb_url"] = artist.get("strArtistThumb")
+
+    return artist_info
+
+
+def lookup_album_from_tadb(name: str, artist: str) -> dict:
+    album_info = {}
+    artist = urllib.parse.quote(artist)
+    name = urllib.parse.quote(name)
+    response = requests.get("".join([ALBUM_SEARCH_URL, artist, "&a=", name]))
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from TADB: {response.status_code}")
+        return {}
+
+    if not response.content:
+        logger.warn(f"Bad content from TADB: {response.content}")
+        return {}
+
+    results = json.loads(response.content)
+    if results["album"]:
+        album = results["album"][0]
+
+        album_info["theaudiodb_id"] = album.get("idAlbum")
+        album_info["theaudiodb_description"] = album.get("strDescriptionEN")
+        album_info["theaudiodb_genre"] = album.get("strGenre")
+        album_info["theaudiodb_style"] = album.get("strStyle")
+        album_info["theaudiodb_mood"] = album.get("strMood")
+        album_info["theaudiodb_speed"] = album.get("strSpeed")
+        album_info["theaudiodb_theme"] = album.get("strTheme")
+        album_info["allmusic_id"] = album.get("strAllMusicID")
+        album_info["wikipedia_slug"] = album.get("strWikipediaID")
+        album_info["discogs_id"] = album.get("strDiscogsID")
+        album_info["wikidata_id"] = album.get("strWikidataID")
+        album_info["rateyourmusic_id"] = album.get("strRateYourMusicID")
+
+        if album.get("intYearReleased"):
+            album_info["theaudiodb_year_released"] = float(
+                album.get("intYearReleased")
+            )
+        if album.get("intScore"):
+            album_info["theaudiodb_score"] = float(album.get("intScore"))
+        if album.get("intScoreVotes"):
+            album_info["theaudiodb_score_votes"] = int(
+                album.get("intScoreVotes")
+            )
+
+    return album_info

vrobbler/apps/music/urls.py

@@ -0,0 +1,26 @@
+from django.urls import path
+from music import views
+
+app_name = "music"
+
+
+urlpatterns = [
+    path("albums/", views.AlbumListView.as_view(), name="albums_list"),
+    path(
+        "album/<slug:slug>/",
+        views.AlbumDetailView.as_view(),
+        name="album_detail",
+    ),
+    path("tracks/", views.TrackListView.as_view(), name="tracks_list"),
+    path(
+        "tracks/<slug:slug>/",
+        views.TrackDetailView.as_view(),
+        name="track_detail",
+    ),
+    path("artists/", views.ArtistListView.as_view(), name="artist_list"),
+    path(
+        "artists/<slug:slug>/",
+        views.ArtistDetailView.as_view(),
+        name="artist_detail",
+    ),
+]

vrobbler/apps/music/utils.py

@@ -0,0 +1,111 @@
+import logging
+import re
+from typing import Optional
+
+from music.musicbrainz import (
+    lookup_album_dict_from_mb,
+    lookup_artist_from_mb,
+    lookup_track_from_mb,
+)
+from music.constants import VARIOUS_ARTIST_DICT
+
+logger = logging.getLogger(__name__)
+
+
+from music.models import Album, Artist, Track
+
+
+def get_or_create_artist(name: str, mbid: str = None) -> Artist:
+    artist = None
+
+    if "feat." in name.lower():
+        name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip()
+    if "featuring" in name.lower():
+        name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip()
+    if "&" in name.lower():
+        name = re.split("&", name, flags=re.IGNORECASE)[0].strip()
+
+    artist_dict = lookup_artist_from_mb(name)
+    mbid = mbid or artist_dict["id"]
+
+    artist = Artist.objects.filter(musicbrainz_id=mbid).first()
+    if not artist:
+        artist = Artist.objects.create(name=name, musicbrainz_id=mbid)
+        artist.fix_metadata()
+
+    return artist
+
+
+def get_or_create_album(
+    name: str, artist: Artist, mbid: str = None
+) -> Optional[Album]:
+    album = None
+    album_dict = lookup_album_dict_from_mb(name, artist_name=artist.name)
+
+    name = name or album_dict["title"]
+    album = Album.objects.filter(artists__in=[artist], name=name).first()
+
+    if not album and name:
+        mbid = mbid or album_dict["mb_id"]
+        album, album_created = Album.objects.get_or_create(
+            name=name, musicbrainz_id=mbid
+        )
+        if album_created:
+            album.year = album_dict["year"]
+            album.musicbrainz_releasegroup_id = album_dict["mb_group_id"]
+            album.musicbrainz_albumartist_id = artist.musicbrainz_id
+            album.save(
+                update_fields=[
+                    "musicbrainz_id",
+                    "year",
+                    "musicbrainz_releasegroup_id",
+                    "musicbrainz_albumartist_id",
+                ]
+            )
+            album.artists.add(artist)
+            album.fix_album_artist()
+            album.fetch_artwork()
+            album.scrape_allmusic()
+
+    if not album:
+        logger.warn(f"No album found for {name} and {mbid}")
+
+    album.fix_album_artist()
+    return album
+
+
+def get_or_create_track(
+    title: str,
+    artist: Artist,
+    album: Album,
+    mbid: str = None,
+    run_time_seconds=None,
+) -> Track:
+    track = None
+    if not mbid:
+        mbid = lookup_track_from_mb(
+            title,
+            artist.musicbrainz_id,
+            album.musicbrainz_id,
+        )["id"]
+
+    track = Track.objects.filter(musicbrainz_id=mbid).first()
+
+    if not track:
+        track = Track.objects.create(
+            title=title,
+            artist=artist,
+            album=album,
+            musicbrainz_id=mbid,
+            run_time_seconds=run_time_seconds,
+        )
+
+    return track
+
+
+def get_or_create_various_artists():
+    artist = Artist.objects.filter(name="Various Artists").first()
+    if not artist:
+        artist = Artist.objects.create(**VARIOUS_ARTIST_DICT)
+        logger.info("Created Various Artists placeholder")
+    return artist

vrobbler/apps/music/views.py

@@ -0,0 +1,94 @@
+from django.db.models import Count
+from django.views import generic
+from music.models import Album, Artist, Track
+from scrobbles.models import ChartRecord
+from scrobbles.stats import get_scrobble_count_qs
+
+
+class TrackListView(generic.ListView):
+    model = Track
+    paginate_by = 200
+
+    def get_queryset(self):
+        return get_scrobble_count_qs(user=self.request.user).order_by(
+            "-scrobble_count"
+        )
+
+
+class TrackDetailView(generic.DetailView):
+    model = Track
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        context_data["charts"] = ChartRecord.objects.filter(
+            track=self.object, rank__in=[1, 2, 3]
+        )
+        return context_data
+
+
+class ArtistListView(generic.ListView):
+    model = Artist
+    paginate_by = 100
+
+    def get_queryset(self):
+        return (
+            super()
+            .get_queryset()
+            .annotate(scrobble_count=Count("track__scrobble"))
+            .order_by("-scrobble_count")
+        )
+
+    def get_context_data(self, *, object_list=None, **kwargs):
+        context_data = super().get_context_data(
+            object_list=object_list, **kwargs
+        )
+        context_data["view"] = self.request.GET.get("view")
+        return context_data
+
+
+class ArtistDetailView(generic.DetailView):
+    model = Artist
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        artist = context_data["object"]
+        rank = 1
+        tracks_ranked = []
+        scrobbles = artist.tracks.first().scrobble_count
+        for track in artist.tracks:
+            if scrobbles > track.scrobble_count:
+                rank += 1
+            tracks_ranked.append((rank, track))
+            scrobbles = track.scrobble_count
+
+        context_data["tracks_ranked"] = tracks_ranked
+        context_data["charts"] = ChartRecord.objects.filter(
+            artist=self.object, rank__in=[1, 2, 3]
+        )
+        return context_data
+
+
+class AlbumListView(generic.ListView):
+    model = Album
+
+    def get_queryset(self):
+        return (
+            super()
+            .get_queryset()
+            .annotate(scrobble_count=Count("track__scrobble"))
+            .order_by("-scrobble_count")
+        )
+
+
+class AlbumDetailView(generic.DetailView):
+    model = Album
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        # context_data['charts'] = ChartRecord.objects.filter(
+        #    track__album=self.object, rank__in=[1, 2, 3]
+        # )
+        return context_data

vrobbler/apps/podcasts/admin.py

@@ -0,0 +1,35 @@
+from django.contrib import admin
+from podcasts.models import Episode, Podcast, Producer
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(Producer)
+class ProducerAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name",)
+    ordering = ("name",)
+
+
+@admin.register(Podcast)
+class PodcastAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "name",
+        "producer",
+        "active",
+    )
+    ordering = ("name",)
+
+
+@admin.register(Episode)
+class EpisodeAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "title",
+        "podcast",
+    )
+    list_filter = ("podcast",)
+    ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/podcasts/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class PodcastsConfig(AppConfig):
+    name = "podcasts"

vrobbler/apps/podcasts/migrations/0001_initial.py

@@ -0,0 +1,158 @@
+# Generated by Django 4.1.5 on 2023-01-12 17:18
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='Producer',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Podcast',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ('active', models.BooleanField(default=True)),
+                ('url', models.URLField(blank=True, null=True)),
+                (
+                    'producer',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='podcasts.producer',
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Episode',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('title', models.CharField(max_length=255)),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'mopidy_uri',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'podcast',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to='podcasts.producer',
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0002_episode_run_time_episode_run_time_ticks_and_more.py

@@ -0,0 +1,32 @@
+# Generated by Django 4.1.5 on 2023-01-12 17:48
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('podcasts', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='episode',
+            name='run_time',
+            field=models.CharField(blank=True, max_length=8, null=True),
+        ),
+        migrations.AddField(
+            model_name='episode',
+            name='run_time_ticks',
+            field=models.PositiveBigIntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='episode',
+            name='podcast',
+            field=models.ForeignKey(
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to='podcasts.podcast',
+            ),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0003_episode_pub_date.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-01-12 18:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('podcasts', '0002_episode_run_time_episode_run_time_ticks_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='episode',
+            name='pub_date',
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0004_episode_number.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-01-12 18:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('podcasts', '0003_episode_pub_date'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='episode',
+            name='number',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0005_alter_episode_options_alter_episode_title.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.5 on 2023-01-13 01:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('podcasts', '0004_episode_number'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='episode',
+            options={},
+        ),
+        migrations.AlterField(
+            model_name='episode',
+            name='title',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0006_alter_episode_run_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0005_alter_episode_options_alter_episode_title"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="episode",
+            name="run_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0007_rename_run_time_episode_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0006_alter_episode_run_time"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="episode",
+            old_name="run_time",
+            new_name="run_time_seconds",
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0008_episode_genre.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-14 22:27
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0033_genre_objectwithgenres"),
+        ("podcasts", "0007_rename_run_time_episode_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="episode",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0009_podcast_cover.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-03-22 22:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0008_episode_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="podcast",
+            name="cover",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="pdocasts/covers/"
+            ),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0010_remove_podcast_cover_podcast_cover_image_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.1.5 on 2023-03-23 02:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0009_podcast_cover"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="podcast",
+            name="cover",
+        ),
+        migrations.AddField(
+            model_name="podcast",
+            name="cover_image",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="podcasts/covers/"
+            ),
+        ),
+        migrations.AddField(
+            model_name="podcast",
+            name="description",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0011_rename_url_podcast_feed_url_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-23 04:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0010_remove_podcast_cover_podcast_cover_image_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="podcast",
+            old_name="url",
+            new_name="feed_url",
+        ),
+        migrations.AddField(
+            model_name="podcast",
+            name="google_podcasts_url",
+            field=models.URLField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/models.py

@@ -0,0 +1,145 @@
+import logging
+from typing import Dict, Optional
+from uuid import uuid4
+
+import requests
+from django.apps import apps
+from django.conf import settings
+from django.core.files.base import ContentFile
+from django.db import models
+from django.urls import reverse
+from django.utils.translation import gettext_lazy as _
+from django_extensions.db.models import TimeStampedModel
+from podcasts.scrapers import scrape_data_from_google_podcasts
+from scrobbles.mixins import ScrobblableMixin
+
+logger = logging.getLogger(__name__)
+BNULL = {"blank": True, "null": True}
+
+
+class Producer(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+
+    def __str__(self):
+        return f"{self.name}"
+
+
+class Podcast(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    producer = models.ForeignKey(
+        Producer, on_delete=models.DO_NOTHING, **BNULL
+    )
+    description = models.TextField(**BNULL)
+    active = models.BooleanField(default=True)
+    feed_url = models.URLField(**BNULL)
+    google_podcasts_url = models.URLField(**BNULL)
+    cover_image = models.ImageField(upload_to="podcasts/covers/", **BNULL)
+
+    def __str__(self):
+        return f"{self.name}"
+
+    def get_absolute_url(self):
+        return reverse("podcasts:podcast_detail", kwargs={"slug": self.uuid})
+
+    def scrobbles(self, user):
+        Scrobble = apps.get_model("scrobbles", "Scrobble")
+        return Scrobble.objects.filter(
+            user=user, podcast_episode__podcast=self
+        ).order_by("-timestamp")
+
+    def scrape_google_podcasts(self, force=False):
+        podcast_dict = {}
+        if not self.cover_image or force:
+            podcast_dict = scrape_data_from_google_podcasts(self.name)
+            if podcast_dict:
+                if not self.producer:
+                    self.producer, created = Producer.objects.get_or_create(
+                        name=podcast_dict["producer"]
+                    )
+                self.description = podcast_dict.get("description")
+                self.google_podcasts_url = podcast_dict.get("google_url")
+                self.save(
+                    update_fields=[
+                        "description",
+                        "producer",
+                        "google_podcasts_url",
+                    ]
+                )
+
+        cover_url = podcast_dict.get("image_url")
+        if (not self.cover_image or force) and cover_url:
+            r = requests.get(cover_url)
+            if r.status_code == 200:
+                fname = f"{self.name}_{self.uuid}.jpg"
+                self.cover_image.save(fname, ContentFile(r.content), save=True)
+
+
+class Episode(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, "PODCAST_COMPLETION_PERCENT", 90)
+
+    podcast = models.ForeignKey(Podcast, on_delete=models.DO_NOTHING)
+    number = models.IntegerField(**BNULL)
+    pub_date = models.DateField(**BNULL)
+    mopidy_uri = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return f"{self.title}"
+
+    @property
+    def subtitle(self):
+        return self.podcast
+
+    @property
+    def info_link(self):
+        return ""
+
+    @property
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.podcast.cover_image:
+            url = self.podcast.cover_image.url
+        return url
+
+    @classmethod
+    def find_or_create(
+        cls, podcast_dict: Dict, producer_dict: Dict, episode_dict: Dict
+    ) -> Optional["Episode"]:
+        """Given a data dict from Mopidy, finds or creates a podcast and
+        producer before saving the epsiode so it can be scrobbled.
+
+        """
+        if not podcast_dict.get("name"):
+            logger.warning(f"No name from source for podcast, not scrobbling")
+            return
+
+        producer = None
+        if producer_dict.get("name"):
+            producer, producer_created = Producer.objects.get_or_create(
+                **producer_dict
+            )
+            if producer_created:
+                logger.debug(f"Created new producer {producer}")
+            else:
+                logger.debug(f"Found producer {producer}")
+
+        if producer:
+            podcast_dict["producer_id"] = producer.id
+        podcast, podcast_created = Podcast.objects.get_or_create(
+            **podcast_dict
+        )
+        if podcast_created:
+            logger.debug(f"Created new podcast {podcast}")
+        else:
+            logger.debug(f"Found podcast {podcast}")
+
+        episode_dict["podcast_id"] = podcast.id
+
+        episode, created = cls.objects.get_or_create(**episode_dict)
+        if created:
+            logger.debug(f"Created new episode: {episode}")
+        else:
+            logger.debug(f"Found episode {episode}")
+
+        return episode

vrobbler/apps/podcasts/scrapers.py

@@ -0,0 +1,91 @@
+import urllib
+from typing import Optional
+from bs4 import BeautifulSoup
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+PODCAST_SEARCH_URL = "https://podcasts.google.com/search/{query}"
+
+
+def _strip_and_clean(text):
+    return text.replace("\n", " ").rstrip().lstrip()
+
+
+def _build_google_url(url):
+    return url.replace("./", "https://podcasts.google.com/")
+
+    return
+
+
+def _get_title_from_soup(soup) -> Optional[int]:
+    title = None
+    try:
+        potential_title = soup.find("div", class_="FyxyKd")
+        if potential_title:
+            title = _strip_and_clean(potential_title.get_text())
+    except ValueError:
+        pass
+    return title
+
+
+def _get_url_from_soup(soup) -> Optional[int]:
+    url = None
+    try:
+        url_tag = soup.find("a", class_="yXo2Qc")
+        if url_tag:
+            url = _build_google_url(url_tag.get("href"))
+    except ValueError:
+        pass
+    return url
+
+
+def _get_producer_from_soup(soup) -> str:
+    pub = ""
+    try:
+        potential_pub = soup.find("div", class_="J3Ov7d")
+        if potential_pub:
+            pub = _strip_and_clean(potential_pub.get_text())
+    except ValueError:
+        pass
+    return pub
+
+
+def _get_description_from_soup(soup) -> str:
+    desc = ""
+    try:
+        potential_desc = soup.find("div", class_="yuTZxb")
+        if potential_desc:
+            desc = _strip_and_clean(potential_desc.get_text())
+    except ValueError:
+        pass
+    return desc
+
+
+def _get_img_url_from_soup(soup) -> str:
+    url = ""
+    try:
+        img_tag = soup.find("img", class_="BhVIWc")
+        try:
+            url = img_tag["src"]
+        except IndexError:
+            pass
+    except ValueError:
+        pass
+    return url
+
+
+def scrape_data_from_google_podcasts(title) -> dict:
+    data_dict = {}
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    url = PODCAST_SEARCH_URL.format(query=title)
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "html.parser")
+        data_dict["title"] = _get_title_from_soup(soup)
+        data_dict["description"] = _get_description_from_soup(soup)
+        data_dict["producer"] = _get_producer_from_soup(soup)
+        data_dict["google_url"] = _get_url_from_soup(soup)
+        data_dict["image_url"] = _get_img_url_from_soup(soup)
+    return data_dict

vrobbler/apps/podcasts/urls.py

@@ -0,0 +1,14 @@
+from django.urls import path
+from podcasts import views
+
+app_name = "podcasts"
+
+
+urlpatterns = [
+    path("podcasts/", views.PodcastListView.as_view(), name="podcast_list"),
+    path(
+        "podcasts/<slug:slug>/",
+        views.PodcastDetailView.as_view(),
+        name="podcast_detail",
+    ),
+]

vrobbler/apps/podcasts/views.py

@@ -0,0 +1,18 @@
+from django.views import generic
+from podcasts.models import Podcast
+
+
+class PodcastListView(generic.ListView):
+    model = Podcast
+    paginate_by = 20
+
+
+class PodcastDetailView(generic.DetailView):
+    model = Podcast
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        user = self.request.user
+        context_data = super().get_context_data(**kwargs)
+        context_data["scrobbles"] = self.object.scrobbles(user)
+        return context_data