Bump version to 0.13.2

This reverts commit c00343abfe.

.drone.yml

@@ -14,6 +14,7 @@ steps:
       # Install dependencies
       - cp vrobbler.conf.test vrobbler.conf
       - pip install poetry
+      - poetry config repositories.unblink "https://pypi.unbl.ink/pypi/simple"
       - poetry install
       # Start with a fresh database (which is already running as a service from Drone)
       - poetry run pytest --cov-report term:skip-covered --cov=vrobbler tests
@@ -23,6 +24,24 @@ steps:
       # Mount pip cache from host
       - name: pip_cache
         path: /root/.cache/pip
+  - name: deploy
+    image: appleboy/drone-ssh
+    settings:
+      host:
+        - vrobbler.service
+      username: root
+      ssh_key:
+        from_secret: ssh_key
+      command_timeout: 2m
+      script:
+        - pip uninstall -y vrobbler
+        - pip install git+https://code.unbl.ink/secstate/vrobbler.git@main
+        - vrobbler migrate
+        - vrobbler collectstatic --noinput
+        - immortalctl restart celery && immortalctl restart vrobbler
+    when:
+      branch:
+      - main
 volumes:
   - name: docker
     host:

Procfile

@@ -0,0 +1,2 @@
+web: python manage.py runserver 0.0.0.0:8014
+worker: celery -A vrobbler worker -l DEBUG

manage.py

@@ -6,7 +6,7 @@ import sys
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'vrobbler.settings')
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "vrobbler.settings")
     try:
         from django.core.management import execute_from_command_line
     except ImportError as exc:
@@ -18,5 +18,5 @@ def main():
     execute_from_command_line(sys.argv)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vrobbler"
-version = "0.9.1"
+version = "0.11.12"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 
@@ -26,7 +26,6 @@ django-taggit = "^2.1.0"
 django-markdownify = "^0.9.1"
 gunicorn = "^20.1.0"
 django-simple-history = "^3.1.1"
-whitenoise = "^6.3.0"
 musicbrainzngs = "^0.7.1"
 cinemagoer = "^2022.12.27"
 pysportsdb = "^0.1.0"
@@ -35,6 +34,12 @@ django-redis = "^5.2.0"
 pylast = "^5.1.0"
 django-encrypted-field = "^1.0.5"
 celery = "^5.2.7"
+honcho = "^1.1.0"
+howlongtobeatpy = "^1.0.5"
+beautifulsoup4 = "^4.11.2"
+django-storages = "^1.13.2"
+boto3 = "^1.26.98"
+stream-sqlite = "^0.0.41"
 
 [tool.poetry.dev-dependencies]
 Werkzeug = "2.0.3"
@@ -62,7 +67,6 @@ DJANGO_SETTINGS_MODULE='vrobbler.settings'
 
 [tool.black]
 line-length = 79
-skip-string-normalization = true
 target-version = ["py39", "py310"]
 include = ".py$"
 exclude = "migrations"

tests/podcasts_tests/test_scrapers.py

@@ -0,0 +1,28 @@
+from vrobbler.apps.podcasts.scrapers import scrape_data_from_google_podcasts
+
+expected_desc = (
+    "NPR's Up First is the news you need to start your day. "
+    "The three biggest stories of the day, with reporting and analysis "
+    "from NPR News — in 10 minutes. Available weekdays by 6 a.m. ET, "
+    "with hosts Leila Fadel, Steve Inskeep, Michel Martin and A Martinez. "
+    "Also available on Saturdays by 8 a.m. ET, with Ayesha Rascoe and "
+    "Scott Simon. On Sundays, hear a longer exploration behind the "
+    "headlines with Rachel Martin, available by 8 a.m. ET. Subscribe "
+    "and listen, then support your local NPR station at donate.npr.org.  "
+    "Support NPR's reporting by subscribing to Up First+ and unlock "
+    "sponsor-free listening. Learn more at plus.npr.org/UpFirst"
+)
+
+expected_img_url = "https://encrypted-tbn2.gstatic.com/images?q=tbn:ANd9GcT-PqtK-bauo8wm8dBE__SVGArlvfBYY8rqxr2kA5UwjKzEx8c"
+expected_google_url = "https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzUxMDMxOC9wb2RjYXN0LnhtbA"
+
+
+def test_get_not_allowed_from_mopidy():
+    query = "Up First"
+    result_dict = scrape_data_from_google_podcasts(query)
+
+    assert result_dict["title"] == query
+    assert result_dict["description"] == expected_desc
+    assert result_dict["image_url"] == expected_img_url
+    assert result_dict["producer"] == "NPR"
+    assert result_dict["google_url"] == expected_google_url

tests/scrobbles_tests/conftest.py

@@ -1,7 +1,6 @@
 import json
 import pytest
 
-from scrobbles.models import Scrobble
 from rest_framework.authtoken.models import Token
 from django.contrib.auth import get_user_model
 
@@ -19,12 +18,12 @@ class MopidyRequest:
     musicbrainz_track_id = "54214d63-5adf-4909-87cd-c65c37a6d558"
     musicbrainz_album_id = "03b864cd-7761-314c-a892-05a89ddff00d"
     musicbrainz_artist_id = "95f5b748-d370-47fe-85bd-0af2dc450bc0"
-    mopidy_uri = "local:track:Sublime%20-%20Sublime/Disc%201%20-%2004%20-%20Same%20in%20the%20End.mp3"
+    mopidy_uri = "local:track:Sublime%20-%20Sublime/Disc%201%20-%2004%20-%20Same%20in%20the%20End.mp3"  # noqa
     status = "resumed"
 
     def __init__(self, **kwargs):
         self.request_data = {
-            "name": kwargs.get('name', self.name),
+            "name": kwargs.get("name", self.name),
             "artist": kwargs.get("artist", self.artist),
             "album": kwargs.get("album", self.album),
             "track_number": int(kwargs.get("track_number", self.track_number)),
@@ -61,7 +60,7 @@ class MopidyRequest:
 
 @pytest.fixture
 def valid_auth_token():
-    user = User.objects.create(email='test@exmaple.com')
+    user = User.objects.create(email="test@exmaple.com")
     return Token.objects.create(user=user).key
 
 

tests/scrobbles_tests/test_aggregators.py

@@ -5,22 +5,17 @@ import time_machine
 from django.contrib.auth import get_user_model
 from django.urls import reverse
 from django.utils import timezone
-from music.aggregators import (
+from music.aggregators import live_charts, scrobble_counts, week_of_scrobbles
-    scrobble_counts,
-    top_artists,
-    top_tracks,
-    week_of_scrobbles,
-)
 from profiles.models import UserProfile
 from scrobbles.models import Scrobble
 
 
 def build_scrobbles(client, request_data, num=7, spacing=2):
-    url = reverse('scrobbles:mopidy-websocket')
+    url = reverse("scrobbles:mopidy-webhook")
-    user = get_user_model().objects.create(username='Test User')
+    user = get_user_model().objects.create(username="Test User")
-    UserProfile.objects.create(user=user, timezone='US/Eastern')
+    UserProfile.objects.create(user=user, timezone="US/Eastern")
     for i in range(num):
-        client.post(url, request_data, content_type='application/json')
+        client.post(url, request_data, content_type="application/json")
         s = Scrobble.objects.last()
         s.user = user
         s.timestamp = timezone.now() - timedelta(days=i * spacing)
@@ -35,11 +30,11 @@ def test_scrobble_counts_data(client, mopidy_track_request_data):
     user = get_user_model().objects.first()
     count_dict = scrobble_counts(user)
     assert count_dict == {
-        'alltime': 7,
+        "alltime": 7,
-        'month': 2,
+        "month": 2,
-        'today': 1,
+        "today": 1,
-        'week': 3,
+        "week": 3,
-        'year': 7,
+        "year": 7,
     }
 
 
@@ -55,7 +50,7 @@ def test_week_of_scrobbles_data(client, mopidy_track_request_data):
 def test_top_tracks_by_day(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
-    tops = top_tracks(user)
+    tops = live_charts(user)
     assert tops[0].title == "Same in the End"
 
 
@@ -63,7 +58,7 @@ def test_top_tracks_by_day(client, mopidy_track_request_data):
 def test_top_tracks_by_week(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
-    tops = top_tracks(user, filter='week')
+    tops = live_charts(user, chart_period="week")
     assert tops[0].title == "Same in the End"
 
 
@@ -71,7 +66,7 @@ def test_top_tracks_by_week(client, mopidy_track_request_data):
 def test_top_tracks_by_month(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
-    tops = top_tracks(user, filter='month')
+    tops = live_charts(user, chart_period="month")
     assert tops[0].title == "Same in the End"
 
 
@@ -79,7 +74,7 @@ def test_top_tracks_by_month(client, mopidy_track_request_data):
 def test_top_tracks_by_year(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
-    tops = top_tracks(user, filter='year')
+    tops = live_charts(user, chart_period="year")
     assert tops[0].title == "Same in the End"
 
 
@@ -87,7 +82,7 @@ def test_top_tracks_by_year(client, mopidy_track_request_data):
 def test_top__artists_by_week(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
-    tops = top_artists(user, filter='week')
+    tops = live_charts(user, chart_period="week", media_type="Artist")
     assert tops[0].name == "Sublime"
 
 
@@ -95,7 +90,7 @@ def test_top__artists_by_week(client, mopidy_track_request_data):
 def test_top__artists_by_month(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
-    tops = top_artists(user, filter='month')
+    tops = live_charts(user, chart_period="month", media_type="Artist")
     assert tops[0].name == "Sublime"
 
 
@@ -103,5 +98,5 @@ def test_top__artists_by_month(client, mopidy_track_request_data):
 def test_top__artists_by_year(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
-    tops = top_artists(user, filter='year')
+    tops = live_charts(user, chart_period="year", media_type="Artist")
     assert tops[0].name == "Sublime"

tests/scrobbles_tests/test_views.py

@@ -1,30 +1,27 @@
-import json
 import pytest
-
 from django.urls import reverse
-
-from scrobbles.models import Scrobble
 from music.models import Track
 from podcasts.models import Episode
+from scrobbles.models import Scrobble
 
 
 @pytest.mark.django_db
 def test_get_not_allowed_from_mopidy(client, valid_auth_token):
-    url = reverse('scrobbles:mopidy-websocket')
+    url = reverse("scrobbles:mopidy-webhook")
-    headers = {'Authorization': f'Token {valid_auth_token}'}
+    headers = {"Authorization": f"Token {valid_auth_token}"}
     response = client.get(url, headers=headers)
     assert response.status_code == 405
 
 
 @pytest.mark.django_db
 def test_bad_mopidy_request_data(client, valid_auth_token):
-    url = reverse('scrobbles:mopidy-websocket')
+    url = reverse("scrobbles:mopidy-webhook")
-    headers = {'Authorization': f'Token {valid_auth_token}'}
+    headers = {"Authorization": f"Token {valid_auth_token}"}
     response = client.post(url, headers)
     assert response.status_code == 400
     assert (
-        response.data['detail']
+        response.data["detail"]
-        == 'JSON parse error - Expecting value: line 1 column 1 (char 0)'
+        == "JSON parse error - Expecting value: line 1 column 1 (char 0)"
     )
 
 
@@ -32,22 +29,23 @@ def test_bad_mopidy_request_data(client, valid_auth_token):
 def test_scrobble_mopidy_track(
     client, mopidy_track_request_data, valid_auth_token
 ):
-    url = reverse('scrobbles:mopidy-websocket')
+    url = reverse("scrobbles:mopidy-webhook")
-    headers = {'Authorization': f'Token {valid_auth_token}'}
+    headers = {"Authorization": f"Token {valid_auth_token}"}
     response = client.post(
         url,
         mopidy_track_request_data,
-        content_type='application/json',
+        content_type="application/json",
         headers=headers,
     )
     assert response.status_code == 200
-    assert response.data == {'scrobble_id': 1}
+    assert response.data == {"scrobble_id": 1}
 
     scrobble = Scrobble.objects.get(id=1)
     assert scrobble.media_obj.__class__ == Track
     assert scrobble.media_obj.title == "Same in the End"
 
 
+@pytest.mark.skip(reason="API is unstable")
 @pytest.mark.django_db
 def test_scrobble_mopidy_same_track_different_album(
     client,
@@ -55,26 +53,28 @@ def test_scrobble_mopidy_same_track_different_album(
     mopidy_track_diff_album_request_data,
     valid_auth_token,
 ):
-    url = reverse('scrobbles:mopidy-websocket')
+    url = reverse("scrobbles:mopidy-webhook")
-    headers = {'Authorization': f'Token {valid_auth_token}'}
+    headers = {"Authorization": f"Token {valid_auth_token}"}
     response = client.post(
         url,
         mopidy_track_request_data,
-        content_type='application/json',
+        content_type="application/json",
         headers=headers,
     )
     assert response.status_code == 200
-    assert response.data == {'scrobble_id': 1}
+    assert response.data == {"scrobble_id": 1}
-    scrobble = Scrobble.objects.get(id=1)
+    scrobble = Scrobble.objects.last()
     assert scrobble.media_obj.album.name == "Sublime"
 
     response = client.post(
         url,
         mopidy_track_diff_album_request_data,
-        content_type='application/json',
+        content_type="application/json",
     )
 
-    scrobble = Scrobble.objects.get(id=2)
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 2}
+    scrobble = Scrobble.objects.last()
     assert scrobble.media_obj.__class__ == Track
     assert scrobble.media_obj.album.name == "Gold"
     assert scrobble.media_obj.title == "Same in the End"
@@ -84,16 +84,16 @@ def test_scrobble_mopidy_same_track_different_album(
 def test_scrobble_mopidy_podcast(
     client, mopidy_podcast_request_data, valid_auth_token
 ):
-    url = reverse('scrobbles:mopidy-websocket')
+    url = reverse("scrobbles:mopidy-webhook")
-    headers = {'Authorization': f'Token {valid_auth_token}'}
+    headers = {"Authorization": f"Token {valid_auth_token}"}
     response = client.post(
         url,
         mopidy_podcast_request_data,
-        content_type='application/json',
+        content_type="application/json",
         headers=headers,
     )
     assert response.status_code == 200
-    assert response.data == {'scrobble_id': 1}
+    assert response.data == {"scrobble_id": 1}
 
     scrobble = Scrobble.objects.get(id=1)
     assert scrobble.media_obj.__class__ == Episode

tests/videos_tests/test_imdb.py

@@ -1,12 +1,11 @@
 import pytest
-import imdb
-from mock import patch
 
-from vrobbler.apps.scrobbles.imdb import lookup_video_from_imdb
+from videos.imdb import lookup_video_from_imdb
 
 
+@pytest.mark.skip(reason="Need to sort out third party API testing")
 def test_lookup_imdb_bad_id(caplog):
-    data = lookup_video_from_imdb('3409324')
+    data = lookup_video_from_imdb("3409324")
     assert data is None
     assert caplog.records[0].levelname == "WARNING"
     assert caplog.records[0].msg == "IMDB ID should begin with 'tt' 3409324"

todos.org

@@ -2,14 +2,81 @@
 
 A fun way to keep track of things in the project to fix or improve.
 
-* DONE [#A] Fix fetching artwork without release group :bug:
+* Version 1.0.0
+** DONE Add a "stop_timestamp" so we don't rely on content length :improvement:scrobbling:
+CLOSED: [2023-04-02 Sun 23:58]
+
+Essentially, we currently have the timestamp as when the content began
+scrobbling and then calculate the finish time from the length of the content.
+This works pretty well because we know how long most things are.
+
+But in some cases, sports events or long podcasts, we may start mid-way through
+an event or finish halfway through but still want to mark it as done. In these
+cases, knowing the finish time could be useful, especially when interfacing with
+other scrobblers which may have different definitions of when a scrobble
+finishes or started.
+** DONE Fix bug with Various Artist albums being labeled with first artist as album artist :scrobbling:bug:music:
+CLOSED: [2023-03-27 Mon 20:18]
+:LOGBOOK:
+CLOCK: [2023-03-26 Sun 22:01]--[2023-03-27 Mon 01:07] =>  3:06
+:END:
+** DONE Fix bug with weekly aggregator being blank on Sundays :aggregators:music:bug:
+CLOSED: [2023-03-26 Sun 13:52]
+** DONE Fix KoReader scrobbling to use pages rather than time of last read :scrobbling:books:improvement:
+CLOSED: [2023-03-26 Sun 13:51]
+:LOGBOOK:
+CLOCK: [2023-03-26 Sun 13:11]--[2023-03-26 Sun 13:51] =>  0:40
+:END:
+** DONE [#A] Add django-storage to store files on S3 :settings:improvement:
+CLOSED: [2023-03-24 Fri 14:46]
+:LOGBOOK:
+CLOCK: [2023-03-24 Fri 10:47]--[2023-03-24 Fri 14:46] =>  3:59
+CLOCK: [2023-03-24 Fri 10:36]--[2023-03-24 Fri 10:40] =>  0:04
+:END:
+** DONE Fix vrobbler settings not using booleans :settings:bug:
+CLOSED: [2023-03-24 Fri 10:45]
+:LOGBOOK:
+CLOCK: [2023-03-24 Fri 10:40]--[2023-03-24 Fri 10:46] =>  0:06
+:END:
+** DONE Update weekly live chart to be 7-day continuous rather than weekly :views:bug:
+CLOSED: [2023-03-24 Fri 00:31]
+The live view will be blank every Monday, no reason to tie it to a day of the
+week. It should be "the last 7 days"
+** DONE [#B] Implement a detail view for TV shows :improvement:views:
+CLOSED: [2023-03-22 Wed 17:05]
+** DONE [#B] Implement a detail view for Movies :improvement:views:
+CLOSED: [2023-03-22 Wed 17:05]
+** DONE Add "service provider" to TV Series, and use that for source when available :bug:scrobbling:
+CLOSED: [2023-03-22 Wed 17:04]
+** DONE Add view for long-play content (books, video games) to restart them :views:improvement:
+CLOSED: [2023-03-22 Wed 17:01]
+** DONE Add live chart view like Maloja :improvement:views:
+CLOSED: [2023-03-07 Tue 11:13]
+** DONE [#C] Figure out how to add to web-scrobbler :improvement:scrobbling:
+CLOSED: [2023-03-22 Wed 17:06]
+
+An example:
+https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
+
+This is actually going to be moot because we can import from LastFM, and
+web-scrobbler integrates well with LastFM. The only thing to think through here
+now is what to do with all the garbage web-scrobbler sometimes pushes to LastFM
+from Youtube (all videos get pushed, sigh).
+
+* Version 0.11.4
+** DONE Add rudimentary video game scrobbling :improvement:content:videogames:
+CLOSED: [2023-03-07 Tue 11:11]
+** DONE Add ability to scrobble from KOReader statistics files :improvement:books:content:
+CLOSED: [2023-03-07 Tue 11:11]
+
+** DONE [#A] Fix fetching artwork without release group :bug:
 CLOSED: [2023-01-29 Sun 14:27]
 
 When we get artwork from Musicbrianz, and it's not found, we should check for
 release groups as well. This will stop issues with missing artwork because of
 obscure MB release matches.
 
-* DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion percent :bug:
+** DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion percent :bug:
 CLOSED: [2023-01-30 Mon 18:31]
 :LOGBOOK:
 CLOCK: [2023-01-30 Mon 18:00]--[2023-01-30 Mon 18:31] =>  0:31
@@ -26,7 +93,7 @@ as complete for the following conditions:
 
 But if we keep listening beyond 90, we should basically ignore updates (or just
 update the existing scrobble)
-* DONE [#A] Add support for Audioscrobbler tab-separated file uploads :improvement:
+** DONE [#A] Add support for Audioscrobbler tab-separated file uploads :improvement:
 CLOSED: [2023-02-03 Fri 16:52]
 
 An example of the format:
@@ -49,22 +116,23 @@ An example of the format:
 311	311	Misdirected Hostility	7	179	S	1740496085	61ff2c1a-fc9c-44c3-8da1-5e50a44245af
 ,
 #+end_src
-* TODO [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
+** DONE [#B] Allow scrobbling music without MB IDs by grabbing them before scrobble :improvement:
-
+CLOSED: [2023-02-17 Fri 00:10]
-Given a UUID from musicbrainz, we should be able to scrobble an album or
-individual track.
-
-* TODO [#A] Add django-storage to store files on S3 :improvement:
-* TODO [#B] Adjust cancel/finish task to use javascript to submit :improvement:
-* TODO [#B] Implement a detail view for TV shows :improvement:
-* TODO [#B] Implement a detail view for Moviews :improvement:
-* TODO [#B] Allow scrobbling music without MB IDs by grabbing them before scrobble :improvement:
 
 This would allow a few nice flows. One, you'd be able to record the play of an
 entire album by just dropping the muscibrainz_id in. This could be helpful for
 offline listening. It would also mean bad metadata from mopidy would not break
 scrobbling.
-* TODO [#C] Implement keeping track of week/month/year chart-toppers :improvement:
+** DONE When updating musicbrainz IDs, clear and run fetch artwrok :improvement:
+CLOSED: [2023-02-17 Fri 00:11]
+** DONE [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
+CLOSED: [2023-03-07 Tue 11:09]
+
+Given a UUID from musicbrainz, we should be able to scrobble an album or
+individual track.
+
+** DONE [#C] Implement keeping track of week/month/year chart-toppers :improvement:
+CLOSED: [2023-03-07 Tue 11:10]
 :LOGBOOK:
 CLOCK: [2023-01-30 Mon 16:30]--[2023-01-30 Mon 18:00] =>  1:30
 :END:
@@ -80,9 +148,10 @@ a period, along with a one, two, three instance.
 Of course, it could also be a data model without a table, where it runs some fun
 calculations, stores it's values in Redis as a long-term lookup table and just
 has to re-populate when the server restarts.
-* TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :improvement:
+* Backlog
-** Example payloads from mopidy-webhooks
+** TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :utility:improvement:
-*** Podcast playback ended
+*** Example payloads from mopidy-webhooks
+**** Podcast playback ended
 #+begin_src json
 {
     "type": "event",
@@ -116,7 +185,7 @@ has to re-populate when the server restarts.
     }
 }
 #+end_src
-*** Podcast playback state changes
+**** Podcast playback state changes
 #+begin_src json
 {
     "type": "event",
@@ -138,7 +207,7 @@ has to re-populate when the server restarts.
     }
 }
 #+end_src
-*** Podcast playback started
+**** Podcast playback started
 #+begin_src json
 {
     "type": "event",
@@ -171,7 +240,7 @@ has to re-populate when the server restarts.
     }
 }
 #+end_src
-*** Podcast playback paused
+**** Podcast playback paused
 #+begin_src  json
 {
     "type": "status",
@@ -202,7 +271,7 @@ has to re-populate when the server restarts.
 }
 
 #+end_src
-*** Track playback started
+**** Track playback started
 #+begin_src json
 {
     "type": "event",
@@ -259,7 +328,7 @@ has to re-populate when the server restarts.
     }
 }
 #+end_src
-*** Track playback in progress
+**** Track playback in progress
 #+begin_src json
 {
     "type": "status",
@@ -313,7 +382,7 @@ has to re-populate when the server restarts.
     }
 }
 #+end_src
-*** Track event playback paused
+**** Track event playback paused
 #+begin_src json
 {
     "type": "event",
@@ -371,10 +440,7 @@ has to re-populate when the server restarts.
     }
 }
 #+end_src
-* TODO [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :improvement:
+** TODO [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :utililty:improvement:
-* TODO Figure out how to add to web-scrobbler :imropvement:
+** TODO What to do with Youtube videos from LastFM and web-scrobbler :bug:source:lastfm:
-
+** TODO Fix bug in Jellyfin scrobbles that spam more scrobbles after completion :scrobbling:videos:bug:
-An example:
+** TODO Fix bug in podcast scrobbling where a second scrobble is created after completion :scrobbling:podcasts:bug:
-https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
-
-* TODO When updating musicbrainz IDs, clear and run fetch artwrok :improvement:

vrobbler.conf.example

@@ -1,11 +1,26 @@
 # You can use this file to set environment variables for your local setup
 #
 VROBBLER_DEBUG=True
-VROBBLER_JSON_LOGGING=True
 VROBBLER_LOG_LEVEL="DEBUG"
+VROBBLER_JSON_LOGGING=True
 VROBBLER_MEDIA_ROOT = "/media/"
-VROBBLER_TMDB_API_KEY = "KEY"
+VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=False
-VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True
 
-VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"
+VROBBLER_USE_S3=False
-VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0"
+# You may also need to set these in your environment
+AWS_S3_ACCESS_KEY_ID=""
+AWS_S3_SECRET_ACCESS_KEY=""
+AWS_S3_CUSTOM_DOMAIN="https://minio.dev/"
+
+# API keys
+VROBBLER_TMDB_API_KEY = "<key>"
+VROBBLER_LASTFM_API_KEY = "<key>"
+VROBBLER_LASTFM_SECRET_KEY = "<key>"
+VROBBLER_THESPORTSDB_API_KEY="<key>"
+VROBBLER_THEAUDIODB_API_KEY="<key>"
+VROBBLER_IGDB_CLIENT_ID="<id>"
+VROBBLER_IGDB_CLIENT_SECRET="<key>"
+
+# Storages
+# VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"
+# VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0"

vrobbler/__init__.py

@@ -2,4 +2,4 @@
 # Django starts so that shared_task will use this app.
 from .celery import app as celery_app
 
-__all__ = ('celery_app',)
+__all__ = ("celery_app",)

vrobbler/apps/books/admin.py

@@ -0,0 +1,42 @@
+from django.contrib import admin
+
+from books.models import Author, Book, Page
+
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(Author)
+class AuthorAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "name",
+        "openlibrary_id",
+        "bio",
+        "wikipedia_url",
+    )
+    ordering = ("-created",)
+    search_fields = ("name",)
+
+
+@admin.register(Page)
+class PageAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_filter = ("book",)
+    ordering = ("book", "number")
+
+
+@admin.register(Book)
+class BookAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "title",
+        "isbn",
+        "first_publish_year",
+        "pages",
+        "openlibrary_id",
+    )
+    search_fields = ("name",)
+    ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/books/api/serializers.py

@@ -0,0 +1,14 @@
+from books.models import Author, Book
+from rest_framework import serializers
+
+
+class AuthorSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Author
+        fields = "__all__"
+
+
+class BookSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Book
+        fields = "__all__"

vrobbler/apps/books/api/views.py

@@ -0,0 +1,19 @@
+from rest_framework import permissions, viewsets
+
+from books.api.serializers import (
+    AuthorSerializer,
+    BookSerializer,
+)
+from books.models import Author, Book
+
+
+class AuthorViewSet(viewsets.ModelViewSet):
+    queryset = Author.objects.all().order_by("-created")
+    serializer_class = AuthorSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class BookViewSet(viewsets.ModelViewSet):
+    queryset = Book.objects.all().order_by("-created")
+    serializer_class = BookSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/books/koreader.py

@@ -0,0 +1,259 @@
+import codecs
+import logging
+import os
+import re
+import sqlite3
+from datetime import datetime
+from enum import Enum
+from typing import Iterable, List
+
+import pytz
+import requests
+from books.models import Author, Book, Page
+from books.openlibrary import get_author_openlibrary_id
+from django.db.models import Sum
+from pylast import httpx, tempfile
+from scrobbles.models import Scrobble
+from stream_sqlite import stream_sqlite
+
+logger = logging.getLogger(__name__)
+
+
+class KoReaderBookColumn(Enum):
+    ID = 0
+    TITLE = 1
+    AUTHORS = 2
+    NOTES = 3
+    LAST_OPEN = 4
+    HIGHLIGHTS = 5
+    PAGES = 6
+    SERIES = 7
+    LANGUAGE = 8
+    MD5 = 9
+    TOTAL_READ_TIME = 10
+    TOTAL_READ_PAGES = 11
+
+
+class KoReaderPageStatColumn(Enum):
+    ID_BOOK = 0
+    PAGE = 1
+    START_TIME = 2
+    DURATION = 3
+    TOTAL_PAGES = 4
+
+
+def _sqlite_bytes(sqlite_url):
+    with httpx.stream("GET", sqlite_url) as r:
+        yield from r.iter_bytes(chunk_size=65_536)
+
+
+def get_book_map_from_sqlite(rows: Iterable) -> dict:
+    """Given an interable of sqlite rows from the books table, lookup existing
+    books, create ones that don't exist, and return a mapping of koreader IDs to
+    primary key IDs for page creation.
+
+    """
+    book_id_map = {}
+
+    for book_row in rows:
+        book = Book.objects.filter(
+            koreader_md5=book_row[KoReaderBookColumn.MD5.value]
+        ).first()
+
+        if not book:
+            book, created = Book.objects.get_or_create(
+                title=book_row[KoReaderBookColumn.TITLE.value]
+            )
+
+            if created:
+                total_pages = book_row[KoReaderBookColumn.PAGES.value]
+                run_time = total_pages * book.AVG_PAGE_READING_SECONDS
+                ko_authors = book_row[
+                    KoReaderBookColumn.AUTHORS.value
+                ].replace("\n", ", ")
+                # Strip middle initials, OpenLibrary often fails with these
+                ko_authors = re.sub(" [A-Z]. ", " ", ko_authors)
+                book_dict = {
+                    "title": book_row[KoReaderBookColumn.TITLE.value],
+                    "pages": total_pages,
+                    "koreader_md5": book_row[KoReaderBookColumn.MD5.value],
+                    "koreader_id": int(book_row[KoReaderBookColumn.ID.value]),
+                    "koreader_authors": ko_authors,
+                    "run_time_seconds": run_time,
+                }
+                Book.objects.filter(pk=book.id).update(**book_dict)
+
+                # Add authors
+                authors = ko_authors.split(", ")
+                author_list = []
+                for author_str in authors:
+                    logger.debug(f"Looking up author {author_str}")
+                    if author_str == "N/A":
+                        continue
+
+                    author, created = Author.objects.get_or_create(
+                        name=author_str
+                    )
+                    if created:
+                        author.openlibrary_id = get_author_openlibrary_id(
+                            author_str
+                        )
+                        author.save(update_fields=["openlibrary_id"])
+                        author.fix_metadata()
+                        logger.debug(f"Created author {author}")
+                    book.authors.add(author)
+
+                # This will try to fix metadata by looking it up on OL
+                book.fix_metadata()
+
+        book.refresh_from_db()
+        total_seconds = 0
+        if book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]:
+            total_seconds = book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]
+
+        book_id_map[book_row[KoReaderBookColumn.ID.value]] = (
+            book.id,
+            total_seconds,
+        )
+
+    return book_id_map
+
+
+def build_scrobbles_from_pages(
+    rows: Iterable, book_id_map: dict, user_id: int
+) -> List[Scrobble]:
+    new_scrobbles = []
+
+    new_scrobbles = []
+    pages_found = []
+    book_read_time_map = {}
+    for page_row in rows:
+        koreader_id = page_row[KoReaderPageStatColumn.ID_BOOK.value]
+        page_number = page_row[KoReaderPageStatColumn.PAGE.value]
+        ts = page_row[KoReaderPageStatColumn.START_TIME.value]
+        book_id = book_id_map[koreader_id][0]
+        book_read_time_map[book_id] = book_id_map[koreader_id][1]
+
+        page, page_created = Page.objects.get_or_create(
+            book_id=book_id, number=page_number, user_id=user_id
+        )
+        if page_created:
+            page.start_time = datetime.utcfromtimestamp(ts).replace(
+                tzinfo=pytz.utc
+            )
+            page.duration_seconds = page_row[
+                KoReaderPageStatColumn.DURATION.value
+            ]
+            page.save(update_fields=["start_time", "duration_seconds"])
+        pages_found.append(page)
+
+    playback_position_seconds = 0
+    for page in set(pages_found):
+        # Add up page seconds to set the aggregate time of all pages to reading time
+        playback_position_seconds = (
+            playback_position_seconds + page.duration_seconds
+        )
+        if page.is_scrobblable:
+            # Check to see if a scrobble with this timestamp, book and user already exists
+            scrobble = Scrobble.objects.filter(
+                timestamp=page.start_time,
+                book_id=page.book_id,
+                user_id=user_id,
+            ).first()
+            if not scrobble:
+                logger.debug(
+                    f"Queueing scrobble for {page.book}, page {page.number}"
+                )
+                new_scrobble = Scrobble(
+                    book_id=page.book_id,
+                    user_id=user_id,
+                    source="KOReader",
+                    timestamp=page.start_time,
+                    played_to_completion=True,
+                    playback_position_seconds=playback_position_seconds,
+                    in_progress=False,
+                    book_pages_read=page.number,
+                    long_play_complete=False,
+                )
+                new_scrobbles.append(new_scrobble)
+            # After setting a scrobblable page, reset our accumulator
+            playback_position_seconds = 0
+    return new_scrobbles
+
+
+def enrich_koreader_scrobbles(scrobbles: list) -> None:
+    """Given a list of scrobbles, update pages read, long play seconds and check
+    for media completion"""
+
+    for scrobble in scrobbles:
+        scrobble.book_pages_read = scrobble.book.page_set.last().number
+        # But if there's a next scrobble, set pages read to their starting page
+        #
+        if scrobble.next:
+            scrobble.book_pages_read = scrobble.next.book_pages_read - 1
+        scrobble.long_play_seconds = scrobble.book.page_set.filter(
+            number__lte=scrobble.book_pages_read
+        ).aggregate(Sum("duration_seconds"))["duration_seconds__sum"]
+
+        scrobble.save(update_fields=["book_pages_read", "long_play_seconds"])
+
+
+def process_koreader_sqlite_url(file_url, user_id) -> list:
+    book_id_map = {}
+    new_scrobbles = []
+
+    for table_name, pragma_table_info, rows in stream_sqlite(
+        _sqlite_bytes(file_url), max_buffer_size=1_048_576
+    ):
+        logger.debug(f"Found table {table_name} - processing")
+        if table_name == "book":
+            book_id_map = get_book_map_from_sqlite(rows)
+
+        if table_name == "page_stat_data":
+            new_scrobbles = build_scrobbles_from_pages(
+                rows, book_id_map, user_id
+            )
+            logger.debug(f"Creating {len(new_scrobbles)} new scrobbles")
+
+    created = []
+    if new_scrobbles:
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        enrich_koreader_scrobbles(created)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={"created_scrobbles": created},
+        )
+    return created
+
+
+def process_koreader_sqlite_file(file_path, user_id) -> list:
+    """Given a sqlite file from KoReader, open the book table, iterate
+    over rows creating scrobbles from each book found"""
+    # Create a SQL connection to our SQLite database
+    con = sqlite3.connect(file_path)
+    cur = con.cursor()
+
+    book_id_map = get_book_map_from_sqlite(cur.execute("SELECT * FROM book"))
+    new_scrobbles = build_scrobbles_from_pages(
+        cur.execute("SELECT * from page_stat_data"), book_id_map, user_id
+    )
+
+    created = []
+    if new_scrobbles:
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        enrich_koreader_scrobbles(created)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={"created_scrobbles": created},
+        )
+    return created
+
+
+def process_koreader_sqlite(file_path: str, user_id: int) -> list:
+    is_os_file = "https://" not in file_path
+
+    if is_os_file:
+        created = process_koreader_sqlite_file(file_path, user_id)
+    else:
+        created = process_koreader_sqlite_url(file_path, user_id)
+    return created

vrobbler/apps/books/migrations/0001_initial.py

@@ -0,0 +1,128 @@
+# Generated by Django 4.1.5 on 2023-02-19 20:17
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='Author',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Book',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'run_time',
+                    models.CharField(blank=True, max_length=8, null=True),
+                ),
+                (
+                    'run_time_ticks',
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                ('title', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'goodreads_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('koreader_id', models.IntegerField(blank=True, null=True)),
+                (
+                    'koreader_authors',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'koreader_md5',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'isbn',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('pages', models.IntegerField(blank=True, null=True)),
+                (
+                    'language',
+                    models.CharField(blank=True, max_length=4, null=True),
+                ),
+                (
+                    'first_publish_year',
+                    models.IntegerField(blank=True, null=True),
+                ),
+                ('authors', models.ManyToManyField(to='books.author')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/books/migrations/0002_book_author_name_book_author_openlibrary_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-06 05:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="author_name",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="author_openlibrary_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0003_book_cover.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-03-06 05:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0002_book_author_name_book_author_openlibrary_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="cover",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="books/covers/"
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0004_remove_book_author_name_and_more.py

@@ -0,0 +1,69 @@
+# Generated by Django 4.1.5 on 2023-03-06 16:31
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0003_book_cover"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="book",
+            name="author_name",
+        ),
+        migrations.RemoveField(
+            model_name="book",
+            name="author_openlibrary_id",
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="headshot",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="books/authors/"
+            ),
+        ),
+        migrations.CreateModel(
+            name="Page",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("number", models.IntegerField()),
+                ("start_time", models.DateTimeField()),
+                ("duration_seconds", models.IntegerField()),
+                (
+                    "book",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="books.book",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("book", "number")},
+            },
+        ),
+    ]

vrobbler/apps/books/migrations/0005_author_uuid.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.5 on 2023-03-06 16:34
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0004_remove_book_author_name_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="uuid",
+            field=models.UUIDField(
+                blank=True, default=uuid.uuid4, editable=False, null=True
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0006_alter_page_duration_seconds_alter_page_start_time.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-06 17:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0005_author_uuid"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="page",
+            name="duration_seconds",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="page",
+            name="start_time",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0007_author_amazon_id_author_bio_author_goodreads_id_and_more.py

@@ -0,0 +1,48 @@
+# Generated by Django 4.1.5 on 2023-03-06 17:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0006_alter_page_duration_seconds_alter_page_start_time"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="amazon_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="bio",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="goodreads_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="isni",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="librarything_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="wikidata_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="wikipedia_url",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0008_book_first_sentence.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.1.5 on 2023-03-06 18:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "books",
+            "0007_author_amazon_id_author_bio_author_goodreads_id_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="first_sentence",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0009_alter_book_run_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0008_book_first_sentence"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="run_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0010_rename_run_time_book_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0009_alter_book_run_time"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="book",
+            old_name="run_time",
+            new_name="run_time_seconds",
+        ),
+    ]

vrobbler/apps/books/migrations/0011_book_genre.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-14 22:27
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0033_genre_objectwithgenres"),
+        ("books", "0010_rename_run_time_book_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0012_page_end_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-03-26 02:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0011_book_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="page",
+            name="end_time",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0013_page_user.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-03-26 05:31
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("books", "0012_page_end_time"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="page",
+            name="user",
+            field=models.ForeignKey(
+                default=1,
+                on_delete=django.db.models.deletion.CASCADE,
+                to=settings.AUTH_USER_MODEL,
+            ),
+            preserve_default=False,
+        ),
+    ]

vrobbler/apps/books/models.py

@@ -0,0 +1,258 @@
+import logging
+from datetime import timedelta
+from uuid import uuid4
+
+import requests
+from books.openlibrary import (
+    lookup_author_from_openlibrary,
+    lookup_book_from_openlibrary,
+)
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.core.files.base import ContentFile
+from django.db import models
+from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
+from scrobbles.mixins import LongPlayScrobblableMixin, ScrobblableMixin
+from scrobbles.utils import get_scrobbles_for_media
+
+logger = logging.getLogger(__name__)
+User = get_user_model()
+BNULL = {"blank": True, "null": True}
+
+
+class Author(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+    headshot = models.ImageField(upload_to="books/authors/", **BNULL)
+    bio = models.TextField(**BNULL)
+    wikipedia_url = models.CharField(max_length=255, **BNULL)
+    isni = models.CharField(max_length=255, **BNULL)
+    wikidata_id = models.CharField(max_length=255, **BNULL)
+    goodreads_id = models.CharField(max_length=255, **BNULL)
+    librarything_id = models.CharField(max_length=255, **BNULL)
+    amazon_id = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return f"{self.name}"
+
+    def fix_metadata(self, data_dict: dict = {}):
+        if not data_dict and self.openlibrary_id:
+            data_dict = lookup_author_from_openlibrary(self.openlibrary_id)
+
+        if not data_dict or not data_dict.get("name"):
+            return
+
+        headshot_url = data_dict.pop("author_headshot_url", "")
+
+        Author.objects.filter(pk=self.id).update(**data_dict)
+        self.refresh_from_db()
+
+        if headshot_url:
+            r = requests.get(headshot_url)
+            if r.status_code == 200:
+                fname = f"{self.name}_{self.uuid}.jpg"
+                self.headshot.save(fname, ContentFile(r.content), save=True)
+
+
+class Book(LongPlayScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, "BOOK_COMPLETION_PERCENT", 95)
+    AVG_PAGE_READING_SECONDS = getattr(
+        settings, "AVERAGE_PAGE_READING_SECONDS", 60
+    )
+
+    title = models.CharField(max_length=255)
+    authors = models.ManyToManyField(Author)
+    goodreads_id = models.CharField(max_length=255, **BNULL)
+    koreader_id = models.IntegerField(**BNULL)
+    koreader_authors = models.CharField(max_length=255, **BNULL)
+    koreader_md5 = models.CharField(max_length=255, **BNULL)
+    isbn = models.CharField(max_length=255, **BNULL)
+    pages = models.IntegerField(**BNULL)
+    language = models.CharField(max_length=4, **BNULL)
+    first_publish_year = models.IntegerField(**BNULL)
+    first_sentence = models.CharField(max_length=255, **BNULL)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+    cover = models.ImageField(upload_to="books/covers/", **BNULL)
+
+    def __str__(self):
+        return f"{self.title} by {self.author}"
+
+    @property
+    def subtitle(self):
+        return f" by {self.author}"
+
+    @property
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.cover:
+            url = self.cover.url
+        return url
+
+    def get_start_url(self):
+        return reverse("scrobbles:start", kwargs={"uuid": self.uuid})
+
+    def get_absolute_url(self):
+        return reverse("books:book_detail", kwargs={"slug": self.uuid})
+
+    def fix_metadata(self, force_update=False):
+        if not self.openlibrary_id or force_update:
+            author_name = ""
+            if self.author:
+                author_name = self.author.name
+            book_dict = lookup_book_from_openlibrary(self.title, author_name)
+            if not book_dict:
+                logger.warn(f"Book not found in OL {self.title}")
+                return
+
+            cover_url = book_dict.pop("cover_url", "")
+            ol_author_id = book_dict.pop("ol_author_id", "")
+            ol_author_name = book_dict.pop("ol_author_name", "")
+            if book_dict.get("pages") == None:
+                book_dict.pop("pages")
+
+            ol_title = book_dict.get("title", "")
+
+            if ol_title.lower() != self.title.lower():
+                logger.warn(
+                    f"OL and KoReader disagree on this book title {self.title} != {ol_title}"
+                )
+
+            Book.objects.filter(pk=self.id).update(**book_dict)
+            self.refresh_from_db()
+
+            if cover_url:
+                r = requests.get(cover_url)
+                if r.status_code == 200:
+                    fname = f"{self.title}_{self.uuid}.jpg"
+                    self.cover.save(fname, ContentFile(r.content), save=True)
+
+            if self.pages:
+                self.run_time_seconds = self.pages * int(
+                    self.AVG_PAGE_READING_SECONDS
+                )
+
+            self.save()
+
+    @property
+    def author(self):
+        return self.authors.first()
+
+    @property
+    def pages_for_completion(self) -> int:
+        if not self.pages:
+            logger.warn(f"{self} has no pages, no completion percentage")
+            return 0
+        return int(self.pages * (self.COMPLETION_PERCENT / 100))
+
+    def update_long_play_seconds(self):
+        """Check page timestamps and duration and update"""
+        if self.page_set.all():
+            ...
+
+    def progress_for_user(self, user_id: int) -> int:
+        """Used to keep track of whether the book is complete or not"""
+        user = User.objects.get(id=user_id)
+        last_scrobble = get_scrobbles_for_media(self, user).last()
+        progress = 0
+        if last_scrobble:
+            progress = int((last_scrobble.book_pages_read / self.pages) * 100)
+        return progress
+
+    @classmethod
+    def find_or_create(cls, data_dict: dict) -> "Game":
+        from books.utils import update_or_create_book
+
+        return update_or_create_book(
+            data_dict.get("title"), data_dict.get("author")
+        )
+
+
+class Page(TimeStampedModel):
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    book = models.ForeignKey(Book, on_delete=models.CASCADE)
+    number = models.IntegerField()
+    start_time = models.DateTimeField(**BNULL)
+    end_time = models.DateTimeField(**BNULL)
+    duration_seconds = models.IntegerField(**BNULL)
+
+    class Meta:
+        unique_together = (
+            "book",
+            "number",
+        )
+
+    def __str__(self):
+        return f"Page {self.number} of {self.book.pages} in {self.book.title}"
+
+    def save(self, *args, **kwargs):
+        if not self.end_time and self.duration_seconds:
+            self._set_end_time()
+
+        return super(Page, self).save(*args, **kwargs)
+
+    @property
+    def next(self):
+        page = self.book.page_set.filter(number=self.number + 1).first()
+        if not page:
+            page = (
+                self.book.page_set.filter(created__gt=self.created)
+                .order_by("created")
+                .first()
+            )
+        return page
+
+    @property
+    def previous(self):
+        page = self.book.page_set.filter(number=self.number - 1).first()
+        if not page:
+            page = (
+                self.book.page_set.filter(created__lt=self.created)
+                .order_by("-created")
+                .first()
+            )
+        return page
+
+    @property
+    def seconds_to_next_page(self) -> int:
+        seconds = 999999  # Effectively infnity time as we have no next
+        if not self.end_time:
+            self._set_end_time()
+        if self.next:
+            seconds = (self.next.start_time - self.end_time).seconds
+        return seconds
+
+    @property
+    def is_scrobblable(self) -> bool:
+        """A page defines the start of a scrobble if the seconds to next page
+        are greater than an hour, or 3600 seconds, and it's not a single page,
+        so the next seconds to next_page is less than an hour as well.
+
+        As a special case, the first recorded page is a scrobble, so we establish
+        when the book was started.
+
+        """
+        is_scrobblable = False
+        over_an_hour_since_last_page = False
+        if not self.previous:
+            is_scrobblable = True
+
+        if self.previous:
+            over_an_hour_since_last_page = (
+                self.previous.seconds_to_next_page >= 3600
+            )
+        blip = self.seconds_to_next_page >= 3600
+
+        if over_an_hour_since_last_page and not blip:
+            is_scrobblable = True
+        return is_scrobblable
+
+    def _set_end_time(self) -> None:
+        if self.end_time:
+            return
+
+        self.end_time = self.start_time + timedelta(
+            seconds=self.duration_seconds
+        )
+        self.save(update_fields=["end_time"])

vrobbler/apps/books/openlibrary.py

@@ -0,0 +1,129 @@
+import json
+import logging
+import urllib
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+ISBN_URL = "https://openlibrary.org/isbn/{isbn}.json"
+SEARCH_URL = "https://openlibrary.org/search.json?q={query}&sort=editions&mode=everything"
+AUTHOR_SEARCH_URL = "https://openlibrary.org/search/authors.json?q={query}"
+COVER_URL = "https://covers.openlibrary.org/b/olid/{id}-L.jpg"
+AUTHOR_URL = "https://openlibrary.org/authors/{id}.json"
+AUTHOR_IMAGE_URL = "https://covers.openlibrary.org/a/olid/{id}-L.jpg"
+
+
+def get_first(key: str, result: dict) -> str:
+    obj = ""
+    if obj_list := result.get(key):
+        obj = obj_list[0]
+    return obj
+
+
+def get_author_openlibrary_id(name: str) -> str:
+    search_url = AUTHOR_SEARCH_URL.format(query=name)
+    response = requests.get(search_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return ""
+
+    results = json.loads(response.content)
+
+    if not results:
+        logger.warn(f"No author results found from search for {name}")
+        return ""
+
+    result = results.get("docs", [])
+    return result[0].get("key")
+
+
+def lookup_author_from_openlibrary(olid: str) -> dict:
+    author_url = AUTHOR_URL.format(id=olid)
+    response = requests.get(author_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return {}
+
+    results = json.loads(response.content)
+
+    if not results:
+        logger.warn(f"No author results found from OL for {olid}")
+        return {}
+
+    remote_ids = results.get("remote_ids", {})
+    bio = ""
+    if results.get("bio"):
+        try:
+            bio = results.get("bio").get("value")
+        except AttributeError:
+            bio = results.get("bio")
+    return {
+        "name": results.get("name"),
+        "openlibrary_id": olid,
+        "wikipedia_url": results.get("wikipedia"),
+        "wikidata_id": remote_ids.get("wikidata"),
+        "isni": remote_ids.get("isni"),
+        "goodreads_id": remote_ids.get("goodreads"),
+        "librarything_id": remote_ids.get("librarything"),
+        "amazon_id": remote_ids.get("amazon"),
+        "bio": bio,
+        "author_headshot_url": AUTHOR_IMAGE_URL.format(id=olid),
+    }
+
+
+def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
+    title_quoted = urllib.parse.quote(title)
+    author_quoted = ""
+    if author:
+        author_quoted = urllib.parse.quote(author)
+    query = f"{title_quoted} {author_quoted}"
+
+    search_url = SEARCH_URL.format(query=query)
+    response = requests.get(search_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return {}
+
+    results = json.loads(response.content)
+
+    if len(results.get("docs")) == 0:
+        logger.warn(f"No results found from OL for {title}")
+        return {}
+
+    top = None
+    for result in results.get("docs"):
+        # These Summary things suck and ruin our one-shot search
+        if "Summary of" not in result.get("title"):
+            top = result
+            break
+
+    if not top:
+        logger.warn(f"No book found for query {query}")
+        return {}
+
+    ol_id = top.get("cover_edition_key")
+    ol_author_id = get_first("author_key", top)
+    first_sentence = ""
+    if top.get("first_sentence"):
+        try:
+            first_sentence = top.get("first_sentence")[0].get("value")
+        except AttributeError:
+            first_sentence = top.get("first_sentence")[0]
+    isbn = None
+    if top.get("isbn"):
+        isbn = top.get("isbn")[0]
+    return {
+        "title": top.get("title"),
+        "isbn": isbn,
+        "openlibrary_id": ol_id,
+        "goodreads_id": get_first("id_goodreads", top),
+        "first_publish_year": top.get("first_publish_year"),
+        "first_sentence": first_sentence,
+        "pages": top.get("number_of_pages_median", None),
+        "cover_url": COVER_URL.format(id=ol_id),
+        "ol_author_id": ol_author_id,
+    }

vrobbler/apps/books/urls.py

@@ -0,0 +1,19 @@
+from django.urls import path
+from books import views
+
+app_name = "books"
+
+
+urlpatterns = [
+    path("book/", views.BookListView.as_view(), name="book_list"),
+    path(
+        "book/<slug:slug>/",
+        views.BookDetailView.as_view(),
+        name="book_detail",
+    ),
+    path(
+        "author/<slug:slug>/",
+        views.AuthorDetailView.as_view(),
+        name="author_detail",
+    ),
+]

vrobbler/apps/books/utils.py

@@ -0,0 +1,52 @@
+from django.core.files.base import ContentFile
+import requests
+from typing import Optional
+from books.openlibrary import (
+    lookup_author_from_openlibrary,
+    lookup_book_from_openlibrary,
+)
+from books.models import Author, Book
+
+
+def update_or_create_book(
+    title: str, author: Optional[str] = None, force_update=False
+) -> Book:
+    book_dict = lookup_book_from_openlibrary(title, author)
+
+    book, book_created = Book.objects.get_or_create(
+        isbn=book_dict.get("isbn"),
+    )
+
+    if book_created or force_update:
+        cover_url = book_dict.pop("cover_url")
+        ol_author_id = book_dict.pop("ol_author_id")
+
+        Book.objects.filter(pk=book.id).update(**book_dict)
+        book.refresh_from_db()
+
+        # Process authors
+        author_dict = lookup_author_from_openlibrary(ol_author_id)
+        author = Author.objects.filter(openlibrary_id=ol_author_id).first()
+        if not author:
+            author_image_url = author_dict.pop("author_headshot_url", None)
+
+            author = Author.objects.create(**author_dict)
+
+            if author_image_url:
+                r = requests.get(author_image_url)
+                if r.status_code == 200:
+                    fname = f"{author.name}_{author.uuid}.jpg"
+                    author.headshot.save(
+                        fname, ContentFile(r.content), save=True
+                    )
+        book.authors.add(author)
+
+        # Process cover URL
+        r = requests.get(cover_url)
+        if r.status_code == 200:
+            fname = f"{book.title}_{book.uuid}.jpg"
+            book.cover.save(fname, ContentFile(r.content), save=True)
+
+        book.fix_metadata()
+
+    return book

vrobbler/apps/books/views.py

@@ -0,0 +1,17 @@
+from django.views import generic
+from books.models import Book, Author
+
+
+class BookListView(generic.ListView):
+    model = Book
+    paginate_by = 20
+
+
+class BookDetailView(generic.DetailView):
+    model = Book
+    slug_field = "uuid"
+
+
+class AuthorDetailView(generic.DetailView):
+    model = Author
+    slug_field = "uuid"

vrobbler/apps/music/admin.py

@@ -8,19 +8,40 @@ from scrobbles.admin import ScrobbleInline
 @admin.register(Album)
 class AlbumAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
-    list_display = ("name", "year", "musicbrainz_id")
+    list_display = (
-    list_filter = ("year",)
+        "name",
-    ordering = ("name",)
+        "year",
+        "album_artist",
+        "theaudiodb_genre",
+        "theaudiodb_mood",
+        "musicbrainz_id",
+    )
+    list_filter = (
+        "theaudiodb_score",
+        "theaudiodb_genre",
+    )
+    ordering = ("-created",)
+    search_fields = ("name",)
     filter_horizontal = [
-        'artists',
+        "artists",
     ]
 
 
 @admin.register(Artist)
 class ArtistAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
-    list_display = ("name", "musicbrainz_id")
+    list_display = (
-    ordering = ("name",)
+        "name",
+        "theaudiodb_mood",
+        "theaudiodb_genre",
+        "musicbrainz_id",
+    )
+    list_filter = (
+        "theaudiodb_mood",
+        "theaudiodb_genre",
+    )
+    search_fields = ("name",)
+    ordering = ("-created",)
 
 
 @admin.register(Track)
@@ -30,10 +51,10 @@ class TrackAdmin(admin.ModelAdmin):
         "title",
         "album",
         "artist",
-        "run_time",
         "musicbrainz_id",
     )
     list_filter = ("album", "artist")
+    search_fields = ("title",)
     ordering = ("-created",)
     inlines = [
         ScrobbleInline,

vrobbler/apps/music/aggregators.py

@@ -1,21 +1,11 @@
 from datetime import datetime, timedelta
-from typing import List, Optional
 
-import pytz
+from django.apps import apps
-from django.db.models import Count, Q, Sum
+from django.db.models import Count, Q, QuerySet
 from django.utils import timezone
-from music.models import Artist, Track
+from profiles.utils import now_user_timezone
 from scrobbles.models import Scrobble
 from videos.models import Video
-from vrobbler.apps.profiles.utils import now_user_timezone
-
-NOW = timezone.now()
-START_OF_TODAY = datetime.combine(NOW.date(), datetime.min.time(), NOW.tzinfo)
-STARTING_DAY_OF_CURRENT_WEEK = NOW.date() - timedelta(
-    days=NOW.today().isoweekday() % 7
-)
-STARTING_DAY_OF_CURRENT_MONTH = NOW.date().replace(day=1)
-STARTING_DAY_OF_CURRENT_YEAR = NOW.date().replace(month=1, day=1)
 
 
 def scrobble_counts(user=None):
@@ -39,23 +29,25 @@ def scrobble_counts(user=None):
         user_filter, played_to_completion=True
     )
     data = {}
-    data['today'] = finished_scrobbles_qs.filter(
+    data["today"] = finished_scrobbles_qs.filter(
         timestamp__gte=start_of_today
     ).count()
-    data['week'] = finished_scrobbles_qs.filter(
+    data["week"] = finished_scrobbles_qs.filter(
         timestamp__gte=starting_day_of_current_week
     ).count()
-    data['month'] = finished_scrobbles_qs.filter(
+    data["month"] = finished_scrobbles_qs.filter(
         timestamp__gte=starting_day_of_current_month
     ).count()
-    data['year'] = finished_scrobbles_qs.filter(
+    data["year"] = finished_scrobbles_qs.filter(
         timestamp__gte=starting_day_of_current_year
     ).count()
-    data['alltime'] = finished_scrobbles_qs.count()
+    data["alltime"] = finished_scrobbles_qs.count()
     return data
 
 
-def week_of_scrobbles(user=None, media: str = 'tracks') -> dict[str, int]:
+def week_of_scrobbles(
+    user=None, start=None, media: str = "tracks"
+) -> dict[str, int]:
 
     now = timezone.now()
     user_filter = Q()
@@ -63,27 +55,26 @@ def week_of_scrobbles(user=None, media: str = 'tracks') -> dict[str, int]:
         now = now_user_timezone(user.profile)
         user_filter = Q(user=user)
 
-    start_of_today = datetime.combine(
+    if not start:
-        now.date(), datetime.min.time(), now.tzinfo
+        start = datetime.combine(now.date(), datetime.min.time(), now.tzinfo)
-    )
 
     scrobble_day_dict = {}
     base_qs = Scrobble.objects.filter(user_filter, played_to_completion=True)
 
     media_filter = Q(track__isnull=False)
-    if media == 'movies':
+    if media == "movies":
         media_filter = Q(video__video_type=Video.VideoType.MOVIE)
-    if media == 'series':
+    if media == "series":
         media_filter = Q(video__video_type=Video.VideoType.TV_EPISODE)
 
     for day in range(6, -1, -1):
-        start = start_of_today - timedelta(days=day)
+        start_day = start - timedelta(days=day)
-        end = datetime.combine(start, datetime.max.time(), now.tzinfo)
+        end = datetime.combine(start_day, datetime.max.time(), now.tzinfo)
-        day_of_week = start.strftime('%A')
+        day_of_week = start_day.strftime("%A")
 
         scrobble_day_dict[day_of_week] = base_qs.filter(
             media_filter,
-            timestamp__gte=start,
+            timestamp__gte=start_day,
             timestamp__lte=end,
             played_to_completion=True,
         ).count()
@@ -91,58 +82,70 @@ def week_of_scrobbles(user=None, media: str = 'tracks') -> dict[str, int]:
     return scrobble_day_dict
 
 
-def top_tracks(
+def live_charts(
-    user: "User", filter: str = "today", limit: int = 15
+    user: "User",
-) -> List["Track"]:
+    media_type: str = "Track",
-
+    chart_period: str = "all",
+    limit: int = 15,
+) -> QuerySet:
     now = timezone.now()
+    tzinfo = now.tzinfo
+    now = now.date()
     if user.is_authenticated:
         now = now_user_timezone(user.profile)
+        tzinfo = now.tzinfo
 
-    start_of_today = datetime.combine(
+    seven_days_ago = now - timedelta(days=7)
-        now.date(), datetime.min.time(), now.tzinfo
+    thirty_days_ago = now - timedelta(days=30)
-    )
+    start_of_today = datetime.combine(now, datetime.min.time(), tzinfo)
-    starting_day_of_current_week = now.date() - timedelta(
+    start_day_of_week = start_of_today - timedelta(
         days=now.today().isoweekday() % 7
     )
-    starting_day_of_current_month = now.date().replace(day=1)
+    start_day_of_month = now.replace(day=1)
-    starting_day_of_current_year = now.date().replace(month=1, day=1)
+    start_day_of_year = now.replace(month=1, day=1)
 
-    time_filter = Q(scrobble__timestamp__gte=start_of_today)
+    media_model = apps.get_model(app_label="music", model_name=media_type)
-    if filter == "week":
-        time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_week)
-    if filter == "month":
-        time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_month)
-    if filter == "year":
-        time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_year)
 
-    return (
+    period_queries = {
-        Track.objects.filter(time_filter)
+        "today": {"scrobble__timestamp__gte": start_of_today},
-        .annotate(num_scrobbles=Count("scrobble", distinct=True))
+        "week": {"scrobble__timestamp__gte": start_day_of_week},
-        .order_by("-num_scrobbles")[:limit]
+        "last7": {"scrobble__timestamp__gte": seven_days_ago},
+        "last30": {"scrobble__timestamp__gte": thirty_days_ago},
+        "month": {"scrobble__timestamp__gte": start_day_of_month},
+        "year": {"scrobble__timestamp__gte": start_day_of_year},
+        "all": {},
+    }
+
+    time_filter = Q()
+    completion_filter = Q(
+        scrobble__user=user, scrobble__played_to_completion=True
     )
+    user_filter = Q(scrobble__user=user)
+    count_field = "scrobble"
 
+    if media_type == "Artist":
+        for period, query_dict in period_queries.items():
+            period_queries[period] = {
+                "track__" + k: v for k, v in query_dict.items()
+            }
+        completion_filter = Q(
+            track__scrobble__user=user,
+            track__scrobble__played_to_completion=True,
+        )
+        count_field = "track__scrobble"
+        user_filter = Q(track__scrobble__user=user)
 
-def top_artists(
+    time_filter = Q(**period_queries[chart_period])
-    user: "User", filter: str = "today", limit: int = 15
-) -> List["Artist"]:
-    time_filter = Q(track__scrobble__timestamp__gte=START_OF_TODAY)
-    if filter == "week":
-        time_filter = Q(
-            track__scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_WEEK
-        )
-    if filter == "month":
-        time_filter = Q(
-            track__scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_MONTH
-        )
-    if filter == "year":
-        time_filter = Q(
-            track__scrobble__timestamp__gte=STARTING_DAY_OF_CURRENT_YEAR
-        )
 
     return (
-        Artist.objects.filter(time_filter)
+        media_model.objects.filter(user_filter, time_filter)
-        .annotate(num_scrobbles=Count("track__scrobble", distinct=True))
+        .annotate(
+            num_scrobbles=Count(
+                count_field,
+                filter=completion_filter,
+                distinct=True,
+            )
+        )
         .order_by("-num_scrobbles")[:limit]
     )
 

vrobbler/apps/music/allmusic.py

@@ -0,0 +1,85 @@
+import urllib
+from typing import Optional
+from bs4 import BeautifulSoup
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+ALLMUSIC_SEARCH_URL = "https://www.allmusic.com/search/{subpath}/{query}"
+
+
+def strip_and_clean(text):
+    return text.strip("\n").rstrip().lstrip()
+
+
+def get_rating_from_soup(soup) -> Optional[int]:
+    rating = None
+    try:
+        potential_rating = soup.find("div", class_="allmusic-rating")
+        if potential_rating:
+            rating = int(strip_and_clean(potential_rating.get_text()))
+    except ValueError:
+        pass
+    return rating
+
+
+def get_review_from_soup(soup) -> str:
+    review = ""
+    try:
+        potential_text = soup.find("div", class_="text")
+        if potential_text:
+            review = strip_and_clean(potential_text.get_text())
+    except ValueError:
+        pass
+    return review
+
+
+def scrape_data_from_allmusic(url) -> dict:
+    data_dict = {}
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "html.parser")
+        data_dict["rating"] = get_rating_from_soup(soup)
+        data_dict["review"] = get_review_from_soup(soup)
+    return data_dict
+
+
+def get_allmusic_slug(artist_name=None, album_name=None) -> str:
+    slug = ""
+    if not artist_name:
+        return slug
+
+    subpath = "artists"
+    class_ = "name"
+    query = urllib.parse.quote(artist_name)
+    if album_name:
+        subpath = "albums"
+        class_ = "title"
+        query = "+".join([query, urllib.parse.quote(album_name)])
+
+    url = ALLMUSIC_SEARCH_URL.format(subpath=subpath, query=query)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+
+    if r.status_code != 200:
+        logger.info(f"Bad http response from Allmusic {r}")
+        return slug
+
+    soup = BeautifulSoup(r.text, "html.parser")
+    results = soup.find("ul", class_="search-results")
+
+    if not results:
+        logger.info(f"No search results for {query}")
+        return slug
+
+    prime_result = results.findAll("div", class_=class_)
+
+    if not prime_result:
+        logger.info(f"Could not find specific result for search {query}")
+
+    result_url = prime_result[0].find_all("a")[0]["href"]
+    slug = result_url.split("/")[-1:][0]
+
+    return slug

vrobbler/apps/music/api/serializers.py

@@ -0,0 +1,20 @@
+from music.models import Album, Artist, Track
+from rest_framework import serializers
+
+
+class ArtistSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Artist
+        fields = "__all__"
+
+
+class AlbumSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Album
+        fields = "__all__"
+
+
+class TrackSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Track
+        fields = "__all__"

vrobbler/apps/music/api/views.py

@@ -0,0 +1,26 @@
+from rest_framework import permissions, viewsets
+
+from music.api.serializers import (
+    TrackSerializer,
+    ArtistSerializer,
+    AlbumSerializer,
+)
+from music.models import Artist, Album, Track
+
+
+class ArtistViewSet(viewsets.ModelViewSet):
+    queryset = Artist.objects.all().order_by("-created")
+    serializer_class = ArtistSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class AlbumViewSet(viewsets.ModelViewSet):
+    queryset = Album.objects.all().order_by("-created")
+    serializer_class = AlbumSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class TrackViewSet(viewsets.ModelViewSet):
+    queryset = Track.objects.all().order_by("-created")
+    serializer_class = TrackSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/music/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class MusicConfig(AppConfig):
-    name = 'music'
+    name = "music"

vrobbler/apps/music/bandcamp.py

@@ -0,0 +1,49 @@
+import logging
+import urllib
+
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+BANDCAMP_SEARCH_URL = "https://bandcamp.com/search?q={query}&item_type={itype}"
+
+
+def get_bandcamp_slug(artist_name=None, album_name=None) -> str:
+    slug = ""
+    if not artist_name:
+        return slug
+
+    query = urllib.parse.quote(artist_name)
+    item_type = "b"
+    class_ = "heading"
+    if album_name:
+        item_type = "a"
+        query = "+".join([query, urllib.parse.quote(album_name)])
+
+    url = BANDCAMP_SEARCH_URL.format(query=query, itype=item_type)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+
+    if r.status_code != 200:
+        logger.info(f"Bad http response from Bandcamp {r}")
+        return slug
+
+    soup = BeautifulSoup(r.text, "html.parser")
+
+    results = soup.find("ul", class_="result-items")
+
+    if not results:
+        logger.info(f"No search results for {query}")
+        return slug
+
+    prime_result = results.findAll("div", class_=class_)
+
+    if not prime_result:
+        logger.info(f"Could not find specific result for search {query}")
+
+    result_url = prime_result[0].find_all("a")[0]["href"]
+    if item_type == "b":
+        slug = result_url.split("/")[2].split(".")[0]
+    else:
+        slug = result_url.split("?")[0]
+    return slug

vrobbler/apps/music/constants.py

@@ -1,16 +1,21 @@
-JELLYFIN_POST_KEYS = {
+VARIOUS_ARTIST_DICT = {
-    'ITEM_TYPE': 'ItemType',
+    "name": "Various Artists",
-    'RUN_TIME_TICKS': 'RunTimeTicks',
+    "theaudiodb_id": "113641",
-    'RUN_TIME': 'RunTime',
+    "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377",
-    'TITLE': 'Name',
+}
-    'TIMESTAMP': 'UtcTimestamp',
+
-    'YEAR': 'Year',
+JELLYFIN_POST_KEYS = {
-    'PLAYBACK_POSITION_TICKS': 'PlaybackPositionTicks',
+    "ITEM_TYPE": "ItemType",
-    'PLAYBACK_POSITION': 'PlaybackPosition',
+    "RUN_TIME": "RunTime",
-    'ARTIST_MB_ID': 'Provider_musicbrainzartist',
+    "TITLE": "Name",
-    'ALBUM_MB_ID': 'Provider_musicbrainzalbum',
+    "TIMESTAMP": "UtcTimestamp",
-    'RELEASEGROUP_MB_ID': 'Provider_musicbrainzreleasegroup',
+    "YEAR": "Year",
-    'TRACK_MB_ID': 'Provider_musicbrainztrack',
+    "PLAYBACK_POSITION_TICKS": "PlaybackPositionTicks",
-    'ALBUM_NAME': 'Album',
+    "PLAYBACK_POSITION": "PlaybackPosition",
-    'ARTIST_NAME': 'Artist',
+    "ARTIST_MB_ID": "Provider_musicbrainzartist",
+    "ALBUM_MB_ID": "Provider_musicbrainzalbum",
+    "RELEASEGROUP_MB_ID": "Provider_musicbrainzreleasegroup",
+    "TRACK_MB_ID": "Provider_musicbrainztrack",
+    "ALBUM_NAME": "Album",
+    "ARTIST_NAME": "Artist",
 }

vrobbler/apps/music/lastfm.py

@@ -47,17 +47,18 @@ class LastFM:
         new_scrobbles = []
         source = "Last.fm"
         source_id = ""
-        latest_scrobbles = self.get_last_scrobbles(time_from=last_processed)
+        lastfm_scrobbles = self.get_last_scrobbles(time_from=last_processed)
 
-        for scrobble in latest_scrobbles:
+        for lfm_scrobble in lastfm_scrobbles:
-            timestamp = scrobble.pop('timestamp')
+            timestamp = lfm_scrobble.pop("timestamp")
 
-            artist = get_or_create_artist(scrobble.pop('artist'))
+            artist = get_or_create_artist(lfm_scrobble.pop("artist"))
-            album = get_or_create_album(scrobble.pop('album'), artist)
+            album = get_or_create_album(lfm_scrobble.pop("album"), artist)
 
-            scrobble['artist'] = artist
+            lfm_scrobble["artist"] = artist
-            scrobble['album'] = album
+            if album:
-            track = get_or_create_track(**scrobble)
+                lfm_scrobble["album"] = album
+            track = get_or_create_track(**lfm_scrobble)
 
             new_scrobble = Scrobble(
                 user=self.vrobbler_user,
@@ -85,31 +86,10 @@ class LastFM:
         created = Scrobble.objects.bulk_create(new_scrobbles)
         logger.info(
             f"Created {len(created)} scrobbles",
-            extra={'created_scrobbles': created},
+            extra={"created_scrobbles": created},
         )
         return created
 
-    @staticmethod
-    def undo_lastfm_import(process_log, dryrun=True):
-        """Given a newline separated list of scrobbles, delete them"""
-        from scrobbles.models import Scrobble
-
-        if not process_log:
-            logger.warning("No lines in process log found to undo")
-            return
-
-        for line in process_log.split('\n'):
-            scrobble_id = line.split("\t")[0]
-            scrobble = Scrobble.objects.filter(id=scrobble_id).first()
-            if not scrobble:
-                logger.warning(
-                    f"Could not find scrobble {scrobble_id} to undo"
-                )
-                continue
-            logger.info(f"Removing scrobble {scrobble_id}")
-            if not dryrun:
-                scrobble.delete()
-
     def get_last_scrobbles(self, time_from=None, time_to=None):
         """Given a user, Last.fm api key, and secret key, grab a list of scrobbled
         tracks"""
@@ -121,20 +101,18 @@ class LastFM:
             lfm_params["time_to"] = int(time_to.timestamp())
 
         # if not time_from and not time_to:
-        lfm_params['limit'] = None
+        lfm_params["limit"] = None
 
         found_scrobbles = self.user.get_recent_tracks(**lfm_params)
         # TOOD spin this out into a celery task over certain threshold of found scrobbles?
 
         for scrobble in found_scrobbles:
             run_time = None
-            run_time_ticks = None
             mbid = None
             artist = None
 
             try:
-                run_time_ticks = scrobble.track.get_duration()
+                run_time = int(scrobble.track.get_duration() / 1000)
-                run_time = int(run_time_ticks / 1000)
                 mbid = scrobble.track.get_mbid()
                 artist = scrobble.track.get_artist().name
             except pylast.MalformedResponseError as e:
@@ -144,8 +122,8 @@ class LastFM:
                     "LastFM barfed trying to get the track for {scrobble.track}"
                 )
 
-            if not mbid or not artist:
+            if not artist:
-                logger.warn(f"Silly LastFM, bad data, bailing on {scrobble}")
+                logger.warn(f"Silly LastFM, no artist found for {scrobble}")
                 continue
 
             timestamp = datetime.utcfromtimestamp(
@@ -159,8 +137,7 @@ class LastFM:
                     "album": scrobble.album,
                     "title": scrobble.track.title,
                     "mbid": mbid,
-                    "run_time": run_time,
+                    "run_time_seconds": run_time,
-                    "run_time_ticks": run_time_ticks,
                     "timestamp": timestamp,
                 }
             )

vrobbler/apps/music/migrations/0010_artist_biography_artist_theaudiodb_genre_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.1.5 on 2023-02-27 03:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0009_alter_track_musicbrainz_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='artist',
+            name='biography',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='artist',
+            name='theaudiodb_genre',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='artist',
+            name='theaudiodb_mood',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0011_artist_thumbnail.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-02-27 04:04
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0010_artist_biography_artist_theaudiodb_genre_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='artist',
+            name='thumbnail',
+            field=models.ImageField(
+                blank=True, null=True, upload_to='artist/'
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0012_album_allmusic_id_album_discogs_id_and_more.py

@@ -0,0 +1,85 @@
+# Generated by Django 4.1.5 on 2023-03-02 19:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0011_artist_thumbnail'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='allmusic_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='discogs_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='rateyourmusic_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_description',
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_genre',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_id',
+            field=models.CharField(
+                blank=True, max_length=255, null=True, unique=True
+            ),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_mood',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_score',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_score_votes',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_speed',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_style',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_theme',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='wikidata_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='album',
+            name='wikipedia_slug',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0013_alter_album_theaudiodb_score.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-02 19:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0012_album_allmusic_id_album_discogs_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='album',
+            name='theaudiodb_score',
+            field=models.FloatField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0014_album_theaudiodb_year_released.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-02 19:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('music', '0013_alter_album_theaudiodb_score'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='album',
+            name='theaudiodb_year_released',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0015_alter_track_run_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0014_album_theaudiodb_year_released"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="track",
+            name="run_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0016_rename_run_time_track_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0015_alter_track_run_time"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="track",
+            old_name="run_time",
+            new_name="run_time_seconds",
+        ),
+    ]

vrobbler/apps/music/migrations/0017_track_genre.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-14 22:27
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0033_genre_objectwithgenres"),
+        ("music", "0016_rename_run_time_track_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="track",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0018_album_allmusic_rating_album_allmusic_review.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-15 01:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0017_track_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="allmusic_rating",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="album",
+            name="allmusic_review",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0019_artist_allmusic_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-15 03:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0018_album_allmusic_rating_album_allmusic_review"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="artist",
+            name="allmusic_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0020_album_bandcamp_id_artist_bandcamp_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-15 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0019_artist_allmusic_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="bandcamp_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name="artist",
+            name="bandcamp_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0021_album_album_artist.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-15 16:22
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0020_album_bandcamp_id_artist_bandcamp_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="album_artist",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                related_name="albums",
+                to="music.artist",
+            ),
+        ),
+    ]

vrobbler/apps/music/migrations/0022_artist_theaudiodb_id.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-03-15 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0021_album_album_artist"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="artist",
+            name="theaudiodb_id",
+            field=models.CharField(
+                blank=True, max_length=255, null=True, unique=True
+            ),
+        ),
+    ]

vrobbler/apps/music/models.py

@@ -1,14 +1,20 @@
 import logging
+from tempfile import NamedTemporaryFile
 from typing import Dict, Optional
+from urllib.request import urlopen
 from uuid import uuid4
 
 import musicbrainzngs
+import requests
 from django.conf import settings
-from django.core.files.base import ContentFile
+from django.core.files.base import ContentFile, File
 from django.db import models
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
+from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic
+from music.bandcamp import get_bandcamp_slug
+from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb
 from scrobbles.mixins import ScrobblableMixin
 
 logger = logging.getLogger(__name__)
@@ -18,10 +24,17 @@ BNULL = {"blank": True, "null": True}
 class Artist(TimeStampedModel):
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     name = models.CharField(max_length=255)
+    biography = models.TextField(**BNULL)
+    theaudiodb_id = models.CharField(max_length=255, unique=True, **BNULL)
+    theaudiodb_genre = models.CharField(max_length=255, **BNULL)
+    theaudiodb_mood = models.CharField(max_length=255, **BNULL)
     musicbrainz_id = models.CharField(max_length=255, **BNULL)
+    allmusic_id = models.CharField(max_length=100, **BNULL)
+    bandcamp_id = models.CharField(max_length=100, **BNULL)
+    thumbnail = models.ImageField(upload_to="artist/", **BNULL)
 
     class Meta:
-        unique_together = [['name', 'musicbrainz_id']]
+        unique_together = [["name", "musicbrainz_id"]]
 
     def __str__(self):
         return self.name
@@ -30,46 +43,189 @@ class Artist(TimeStampedModel):
     def mb_link(self):
         return f"https://musicbrainz.org/artist/{self.musicbrainz_id}"
 
+    @property
+    def allmusic_link(self):
+        if self.allmusic_id:
+            return f"https://www.allmusic.com/artist/{self.allmusic_id}"
+        return ""
+
+    @property
+    def bandcamp_link(self):
+        if self.bandcamp_id:
+            return f"https://{self.bandcamp_id}.bandcamp.com/"
+        return ""
+
     def get_absolute_url(self):
-        return reverse('music:artist_detail', kwargs={'slug': self.uuid})
+        return reverse("music:artist_detail", kwargs={"slug": self.uuid})
 
     def scrobbles(self):
         from scrobbles.models import Scrobble
 
         return Scrobble.objects.filter(
             track__in=self.track_set.all()
-        ).order_by('-timestamp')
+        ).order_by("-timestamp")
 
     @property
     def tracks(self):
         return (
             self.track_set.all()
-            .annotate(scrobble_count=models.Count('scrobble'))
+            .annotate(scrobble_count=models.Count("scrobble"))
-            .order_by('-scrobble_count')
+            .order_by("-scrobble_count")
         )
 
     def charts(self):
         from scrobbles.models import ChartRecord
 
-        return ChartRecord.objects.filter(track__artist=self).order_by('-year')
+        return ChartRecord.objects.filter(track__artist=self).order_by("-year")
+
+    def scrape_allmusic(self, force=False) -> None:
+        if not self.allmusic_id or force:
+            slug = get_allmusic_slug(self.name)
+            if not slug:
+                logger.info(f"No allmsuic link for {self}")
+                return
+            self.allmusic_id = slug
+            self.save(update_fields=["allmusic_id"])
+
+    def scrape_bandcamp(self, force=False) -> None:
+        if not self.bandcamp_id or force:
+            slug = get_bandcamp_slug(self.name)
+            if not slug:
+                logger.info(f"No bandcamp link for {self}")
+                return
+            self.bandcamp_id = slug
+            self.save(update_fields=["bandcamp_id"])
+
+    def fix_metadata(self):
+        tadb_info = lookup_artist_from_tadb(self.name)
+        if not tadb_info:
+            logger.warn(f"No response from TADB for artist {self.name}")
+            return
+
+        self.biography = tadb_info["biography"]
+        self.theaudiodb_genre = tadb_info["genre"]
+        self.theaudiodb_mood = tadb_info["mood"]
+
+        thumb_url = tadb_info.get("thumb_url", "")
+        if thumb_url:
+            r = requests.get(thumb_url)
+            if r.status_code == 200:
+                fname = f"{self.name}_{self.uuid}.jpg"
+                self.thumbnail.save(fname, ContentFile(r.content), save=True)
+
+    @property
+    def rym_link(self):
+        artist_slug = self.name.lower().replace(" ", "-")
+        return f"https://rateyourmusic.com/artist/{artist_slug}/"
+
+    @property
+    def bandcamp_search_link(self):
+        artist = self.name.lower()
+        return f"https://bandcamp.com/search?q={artist}&item_type=b"
 
 
 class Album(TimeStampedModel):
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     name = models.CharField(max_length=255)
+    album_artist = models.ForeignKey(
+        Artist, related_name="albums", on_delete=models.DO_NOTHING, **BNULL
+    )
     artists = models.ManyToManyField(Artist)
     year = models.IntegerField(**BNULL)
     musicbrainz_id = models.CharField(max_length=255, unique=True, **BNULL)
     musicbrainz_releasegroup_id = models.CharField(max_length=255, **BNULL)
     musicbrainz_albumartist_id = models.CharField(max_length=255, **BNULL)
     cover_image = models.ImageField(upload_to="albums/", **BNULL)
+    theaudiodb_id = models.CharField(max_length=255, unique=True, **BNULL)
+    theaudiodb_description = models.TextField(**BNULL)
+    theaudiodb_year_released = models.IntegerField(**BNULL)
+    theaudiodb_score = models.FloatField(**BNULL)
+    theaudiodb_score_votes = models.IntegerField(**BNULL)
+    theaudiodb_genre = models.CharField(max_length=255, **BNULL)
+    theaudiodb_style = models.CharField(max_length=255, **BNULL)
+    theaudiodb_mood = models.CharField(max_length=255, **BNULL)
+    theaudiodb_speed = models.CharField(max_length=255, **BNULL)
+    theaudiodb_theme = models.CharField(max_length=255, **BNULL)
+    allmusic_id = models.CharField(max_length=255, **BNULL)
+    allmusic_rating = models.IntegerField(**BNULL)
+    allmusic_review = models.TextField(**BNULL)
+    bandcamp_id = models.CharField(max_length=100, **BNULL)
+    rateyourmusic_id = models.CharField(max_length=255, **BNULL)
+    wikipedia_slug = models.CharField(max_length=255, **BNULL)
+    discogs_id = models.CharField(max_length=255, **BNULL)
+    wikidata_id = models.CharField(max_length=255, **BNULL)
 
     def __str__(self):
         return self.name
 
+    def get_absolute_url(self):
+        return reverse("music:album_detail", kwargs={"slug": self.uuid})
+
+    def scrobbles(self):
+        from scrobbles.models import Scrobble
+
+        return Scrobble.objects.filter(
+            track__in=self.track_set.all()
+        ).order_by("-timestamp")
+
     @property
-    def primary_artist(self):
+    def tracks(self):
-        return self.artists.first()
+        return (
+            self.track_set.all()
+            .annotate(scrobble_count=models.Count("scrobble"))
+            .order_by("-scrobble_count")
+        )
+
+    def fix_album_artist(self):
+        from music.utils import get_or_create_various_artists
+
+        multiple_artists = self.artists.count() > 1
+        if multiple_artists:
+            self.album_artist = get_or_create_various_artists()
+        else:
+            self.album_artist = self.artists.first()
+        self.save(update_fields=["album_artist"])
+
+    def scrape_allmusic(self, force=False) -> None:
+        if not self.allmusic_id or force:
+            slug = get_allmusic_slug(self.name, self.album_artist.name)
+            if not slug:
+                logger.info(
+                    f"No allmsuic link for {self} by {self.album_artist}"
+                )
+                return
+            self.allmusic_id = slug
+            self.save(update_fields=["allmusic_id"])
+
+        allmusic_data = scrape_data_from_allmusic(self.allmusic_link)
+
+        if not allmusic_data:
+            logger.info(f"No allmsuic data for {self} by {self.album_artist}")
+            return
+
+        self.allmusic_review = allmusic_data["review"]
+        self.allmusic_rating = allmusic_data["rating"]
+        self.save(update_fields=["allmusic_review", "allmusic_rating"])
+
+    def scrape_theaudiodb(self) -> None:
+        artist = "Various Artists"
+        if self.album_artist:
+            artist = self.album_artist.name
+        album_data = lookup_album_from_tadb(self.name, artist)
+        if not album_data.get("theaudiodb_id"):
+            logger.info(f"No data for {self} found in TheAudioDB")
+            return
+
+        Album.objects.filter(pk=self.pk).update(**album_data)
+
+    def scrape_bandcamp(self, force=False) -> None:
+        if not self.bandcamp_id or force:
+            slug = get_bandcamp_slug(self.album_artist.name, self.name)
+            if not slug:
+                logger.info(f"No bandcamp link for {self}")
+                return
+            self.bandcamp_id = slug
+            self.save(update_fields=["bandcamp_id"])
 
     def fix_metadata(self):
         if (
@@ -77,21 +233,21 @@ class Album(TimeStampedModel):
             or not self.year
             or not self.musicbrainz_releasegroup_id
         ):
-            musicbrainzngs.set_useragent('vrobbler', '0.3.0')
+            musicbrainzngs.set_useragent("vrobbler", "0.3.0")
             mb_data = musicbrainzngs.get_release_by_id(
-                self.musicbrainz_id, includes=['artists', 'release-groups']
+                self.musicbrainz_id, includes=["artists", "release-groups"]
             )
             if not self.musicbrainz_releasegroup_id:
-                self.musicbrainz_releasegroup_id = mb_data['release'][
+                self.musicbrainz_releasegroup_id = mb_data["release"][
-                    'release-group'
+                    "release-group"
-                ]['id']
+                ]["id"]
             if not self.musicbrainz_albumartist_id:
-                self.musicbrainz_albumartist_id = mb_data['release'][
+                self.musicbrainz_albumartist_id = mb_data["release"][
-                    'artist-credit'
+                    "artist-credit"
-                ][0]['artist']['id']
+                ][0]["artist"]["id"]
             if not self.year:
                 try:
-                    self.year = mb_data['release']['date'][0:4]
+                    self.year = mb_data["release"]["date"][0:4]
                 except KeyError:
                     pass
                 except IndexError:
@@ -99,9 +255,9 @@ class Album(TimeStampedModel):
 
             self.save(
                 update_fields=[
-                    'musicbrainz_albumartist_id',
+                    "musicbrainz_albumartist_id",
-                    'musicbrainz_releasegroup_id',
+                    "musicbrainz_releasegroup_id",
-                    'year',
+                    "year",
                 ]
             )
 
@@ -115,9 +271,12 @@ class Album(TimeStampedModel):
                     self.artists.add(t.artist)
             if (
                 not self.cover_image
-                or self.cover_image == 'default-image-replace-me'
+                or self.cover_image == "default-image-replace-me"
             ):
                 self.fetch_artwork()
+        self.fix_album_artist()
+        self.scrape_theaudiodb()
+        self.scrape_allmusic()
 
     def fetch_artwork(self, force=False):
         if not self.cover_image and not force:
@@ -128,10 +287,10 @@ class Album(TimeStampedModel):
                     )
                     name = f"{self.name}_{self.uuid}.jpg"
                     self.cover_image = ContentFile(img_data, name=name)
-                    logger.info(f'Setting image to {name}')
+                    logger.info(f"Setting image to {name}")
                 except musicbrainzngs.ResponseError:
                     logger.warning(
-                        f'No cover art found for {self.name} by release'
+                        f"No cover art found for {self.name} by release"
                     )
 
             if (
@@ -144,10 +303,10 @@ class Album(TimeStampedModel):
                     )
                     name = f"{self.name}_{self.uuid}.jpg"
                     self.cover_image = ContentFile(img_data, name=name)
-                    logger.info(f'Setting image to {name}')
+                    logger.info(f"Setting image to {name}")
                 except musicbrainzngs.ResponseError:
                     logger.warning(
-                        f'No cover art found for {self.name} by release group'
+                        f"No cover art found for {self.name} by release group"
                     )
             if not self.cover_image:
                 logger.debug(
@@ -156,35 +315,88 @@ class Album(TimeStampedModel):
             self.save()
 
     @property
-    def mb_link(self):
+    def mb_link(self) -> str:
         return f"https://musicbrainz.org/release/{self.musicbrainz_id}"
 
+    @property
+    def allmusic_link(self) -> str:
+        if self.allmusic_id:
+            return f"https://www.allmusic.com/album/{self.allmusic_id}"
+        return ""
+
+    @property
+    def wikipedia_link(self):
+        if self.wikipedia_slug:
+            return f"https://www.wikipedia.org/en/{self.wikipedia_slug}"
+        return ""
+
+    @property
+    def tadb_link(self):
+        if self.theaudiodb_id:
+            return f"https://www.theaudiodb.com/album/{self.theaudiodb_id}"
+        return ""
+
+    @property
+    def rym_link(self):
+        artist_slug = self.album_artist.name.lower().replace(" ", "-")
+        album_slug = self.name.lower().replace(" ", "-")
+        return f"https://rateyourmusic.com/release/album/{artist_slug}/{album_slug}/"
+
+    @property
+    def bandcamp_link(self):
+        if self.bandcamp_id and self.album_artist.bandcamp_id:
+            return f"https://{self.album_artist.bandcamp_id}.bandcamp.com/album/{self.bandcamp_id}"
+        return ""
+
+    @property
+    def bandcamp_search_link(self):
+        artist = self.album_artist.name.lower()
+        album = self.name.lower()
+        return f"https://bandcamp.com/search?q={album} {artist}&item_type=a"
+
 
 class Track(ScrobblableMixin):
-    COMPLETION_PERCENT = getattr(settings, 'MUSIC_COMPLETION_PERCENT', 90)
+    COMPLETION_PERCENT = getattr(settings, "MUSIC_COMPLETION_PERCENT", 90)
 
     class Opinion(models.IntegerChoices):
-        DOWN = -1, 'Thumbs down'
+        DOWN = -1, "Thumbs down"
-        NEUTRAL = 0, 'No opinion'
+        NEUTRAL = 0, "No opinion"
-        UP = 1, 'Thumbs up'
+        UP = 1, "Thumbs up"
 
     artist = models.ForeignKey(Artist, on_delete=models.DO_NOTHING)
     album = models.ForeignKey(Album, on_delete=models.DO_NOTHING, **BNULL)
     musicbrainz_id = models.CharField(max_length=255, **BNULL)
 
     class Meta:
-        unique_together = [['album', 'musicbrainz_id']]
+        unique_together = [["album", "musicbrainz_id"]]
 
     def __str__(self):
         return f"{self.title} by {self.artist}"
 
     def get_absolute_url(self):
-        return reverse('music:track_detail', kwargs={'slug': self.uuid})
+        return reverse("music:track_detail", kwargs={"slug": self.uuid})
+
+    @property
+    def subtitle(self):
+        return self.artist
 
     @property
     def mb_link(self):
         return f"https://musicbrainz.org/recording/{self.musicbrainz_id}"
 
+    @property
+    def info_link(self):
+        return self.mb_link
+
+    @property
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.artist.thumbnail:
+            url = self.artist.thumbnail.url
+        if self.album and self.album.cover_image:
+            url = self.album.cover_image.url
+        return url
+
     @classmethod
     def find_or_create(
         cls, artist_dict: Dict, album_dict: Dict, track_dict: Dict
@@ -194,8 +406,8 @@ class Track(ScrobblableMixin):
         exist.
 
         """
-        if not artist_dict.get('name') or not artist_dict.get(
+        if not artist_dict.get("name") or not artist_dict.get(
-            'musicbrainz_id'
+            "musicbrainz_id"
         ):
             logger.warning(
                 f"No artist or artist musicbrainz ID found in message from source, not scrobbling"
@@ -209,8 +421,8 @@ class Track(ScrobblableMixin):
         if not album.cover_image:
             album.fetch_artwork()
 
-        track_dict['album_id'] = getattr(album, "id", None)
+        track_dict["album_id"] = getattr(album, "id", None)
-        track_dict['artist_id'] = artist.id
+        track_dict["artist_id"] = artist.id
 
         track, created = cls.objects.get_or_create(**track_dict)
 

vrobbler/apps/music/musicbrainz.py

@@ -0,0 +1,110 @@
+import logging
+
+import musicbrainzngs
+from dateutil.parser import parse
+
+logger = logging.getLogger(__name__)
+
+
+def lookup_album_from_mb(musicbrainz_id: str) -> dict:
+    release_dict = {}
+
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+    release_data = musicbrainzngs.get_release_by_id(
+        musicbrainz_id,
+        includes=["artists", "release-groups", "recordings"],
+    ).get("release")
+
+    if not release_data:
+        return release_dict
+
+    primary_artist = release_data.get("artist-credit")[0]
+    release_dict = {
+        "artist": {
+            "name": primary_artist.get("name"),
+            "musicbrainz_id": primary_artist.get("id"),
+        },
+        "album": {
+            "name": release_data.get("title"),
+            "musicbrainz_id": musicbrainz_id,
+            "musicbrainz_releasegroup_id": release_data.get(
+                "release-group"
+            ).get("id"),
+            "musicbrainz_albumaritist_id": primary_artist.get("id"),
+            "year": release_data.get("year")[0:4],
+        },
+    }
+
+    release_dict["tracks"] = []
+    for track in release_data.get("medium-list")[0]["track-list"]:
+        recording = track["recording"]
+        release_dict["tracks"].append(
+            {
+                "title": recording["title"],
+                "musicbrainz_id": recording["id"],
+                "run_time": track["length"] / 1000,
+            }
+        )
+
+    return release_dict
+
+
+def lookup_album_dict_from_mb(release_name: str, artist_name: str) -> dict:
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+
+    top_result = musicbrainzngs.search_releases(
+        release_name, artist=artist_name
+    )["release-list"][0]
+    score = int(top_result.get("ext:score"))
+    if score < 85:
+        logger.debug(
+            "Album lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return {}
+    year = None
+    if top_result.get("date"):
+        year = parse(top_result["date"]).year
+
+    return {
+        "year": year,
+        "title": top_result["title"],
+        "mb_id": top_result["id"],
+        "mb_group_id": top_result["release-group"]["id"],
+    }
+
+
+def lookup_artist_from_mb(artist_name: str) -> str:
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+
+    top_result = musicbrainzngs.search_artists(artist=artist_name)[
+        "artist-list"
+    ][0]
+    score = int(top_result.get("ext:score"))
+    if score < 85:
+        logger.debug(
+            "Artist lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return ""
+
+    return top_result
+
+
+def lookup_track_from_mb(
+    track_name: str, artist_mbid: str, album_mbid: str
+) -> str:
+    musicbrainzngs.set_useragent("vrobbler", "0.3.0")
+
+    top_result = musicbrainzngs.search_recordings(
+        query=track_name, artist=artist_mbid, release=album_mbid
+    )["recording-list"][0]
+    score = int(top_result.get("ext:score"))
+    if score < 85:
+        logger.debug(
+            "Track lookup score below 85 threshold",
+            extra={"result": top_result},
+        )
+        return ""
+
+    return top_result

vrobbler/apps/music/serializers.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

vrobbler/apps/music/theaudiodb.py

@@ -0,0 +1,82 @@
+import urllib
+import json
+import logging
+
+import requests
+from django.conf import settings
+
+THEAUDIODB_API_KEY = getattr(settings, "THEAUDIODB_API_KEY")
+ARTIST_SEARCH_URL = f"https://www.theaudiodb.com/api/v1/json/{THEAUDIODB_API_KEY}/search.php?s="
+ALBUM_SEARCH_URL = f"https://www.theaudiodb.com/api/v1/json/{THEAUDIODB_API_KEY}/searchalbum.php?s="
+
+logger = logging.getLogger(__name__)
+
+
+def lookup_artist_from_tadb(name: str) -> dict:
+    artist_info = {}
+    name = urllib.parse.quote(name)
+    response = requests.get(ARTIST_SEARCH_URL + name)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from TADB: {response.status_code}")
+        return {}
+
+    if not response.content:
+        logger.warn(f"Bad content from TADB: {response.content}")
+        return {}
+
+    results = json.loads(response.content)
+    if results["artists"]:
+        artist = results["artists"][0]
+
+        artist_info["biography"] = artist.get("strBiographyEN")
+        artist_info["genre"] = artist.get("strGenre")
+        artist_info["mood"] = artist.get("strMood")
+        artist_info["thumb_url"] = artist.get("strArtistThumb")
+
+    return artist_info
+
+
+def lookup_album_from_tadb(name: str, artist: str) -> dict:
+    album_info = {}
+    artist = urllib.parse.quote(artist)
+    name = urllib.parse.quote(name)
+    response = requests.get("".join([ALBUM_SEARCH_URL, artist, "&a=", name]))
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from TADB: {response.status_code}")
+        return {}
+
+    if not response.content:
+        logger.warn(f"Bad content from TADB: {response.content}")
+        return {}
+
+    results = json.loads(response.content)
+    if results["album"]:
+        album = results["album"][0]
+
+        album_info["theaudiodb_id"] = album.get("idAlbum")
+        album_info["theaudiodb_description"] = album.get("strDescriptionEN")
+        album_info["theaudiodb_genre"] = album.get("strGenre")
+        album_info["theaudiodb_style"] = album.get("strStyle")
+        album_info["theaudiodb_mood"] = album.get("strMood")
+        album_info["theaudiodb_speed"] = album.get("strSpeed")
+        album_info["theaudiodb_theme"] = album.get("strTheme")
+        album_info["allmusic_id"] = album.get("strAllMusicID")
+        album_info["wikipedia_slug"] = album.get("strWikipediaID")
+        album_info["discogs_id"] = album.get("strDiscogsID")
+        album_info["wikidata_id"] = album.get("strWikidataID")
+        album_info["rateyourmusic_id"] = album.get("strRateYourMusicID")
+
+        if album.get("intYearReleased"):
+            album_info["theaudiodb_year_released"] = float(
+                album.get("intYearReleased")
+            )
+        if album.get("intScore"):
+            album_info["theaudiodb_score"] = float(album.get("intScore"))
+        if album.get("intScoreVotes"):
+            album_info["theaudiodb_score_votes"] = int(
+                album.get("intScoreVotes")
+            )
+
+    return album_info

vrobbler/apps/music/urls.py

@@ -1,21 +1,26 @@
 from django.urls import path
 from music import views
 
-app_name = 'music'
+app_name = "music"
 
 
 urlpatterns = [
-    path('albums/', views.AlbumListView.as_view(), name='albums_list'),
+    path("albums/", views.AlbumListView.as_view(), name="albums_list"),
-    path("tracks/", views.TrackListView.as_view(), name='tracks_list'),
     path(
-        'tracks/<slug:slug>/',
+        "album/<slug:slug>/",
-        views.TrackDetailView.as_view(),
+        views.AlbumDetailView.as_view(),
-        name='track_detail',
+        name="album_detail",
     ),
-    path('artists/', views.ArtistListView.as_view(), name='artist_list'),
+    path("tracks/", views.TrackListView.as_view(), name="tracks_list"),
     path(
-        'artists/<slug:slug>/',
+        "tracks/<slug:slug>/",
+        views.TrackDetailView.as_view(),
+        name="track_detail",
+    ),
+    path("artists/", views.ArtistListView.as_view(), name="artist_list"),
+    path(
+        "artists/<slug:slug>/",
         views.ArtistDetailView.as_view(),
-        name='artist_detail',
+        name="artist_detail",
     ),
 ]

vrobbler/apps/music/utils.py

@@ -1,106 +1,111 @@
-import re
 import logging
+import re
+from typing import Optional
 
-from scrobbles.musicbrainz import (
+from music.musicbrainz import (
     lookup_album_dict_from_mb,
     lookup_artist_from_mb,
+    lookup_track_from_mb,
 )
+from music.constants import VARIOUS_ARTIST_DICT
 
 logger = logging.getLogger(__name__)
 
 
-from music.models import Artist, Album, Track
+from music.models import Album, Artist, Track
 
 
 def get_or_create_artist(name: str, mbid: str = None) -> Artist:
     artist = None
-    logger.debug(f'Got artist {name} and mbid: {mbid}')
 
-    if 'feat.' in name.lower():
+    if "feat." in name.lower():
         name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip()
-    if 'featuring' in name.lower():
+    if "featuring" in name.lower():
         name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip()
+    if "&" in name.lower():
+        name = re.split("&", name, flags=re.IGNORECASE)[0].strip()
+
     artist_dict = lookup_artist_from_mb(name)
-    mbid = mbid or artist_dict['id']
+    mbid = mbid or artist_dict["id"]
 
-    logger.debug(f'Looking up artist {name} and mbid: {mbid}')
+    artist = Artist.objects.filter(musicbrainz_id=mbid).first()
-    artist, created = Artist.objects.get_or_create(
+    if not artist:
-        name=name, musicbrainz_id=mbid
+        artist = Artist.objects.create(name=name, musicbrainz_id=mbid)
-    )
+        artist.fix_metadata()
-
-    logger.debug(f"Cleaning artist {name} with {artist_dict['name']}")
-    # Clean up bad names in our DB with MB names
-    if artist.name != artist_dict['name']:
-        artist.name = artist_dict["name"]
-        artist.save(update_fields=["name"])
 
     return artist
 
 
-def get_or_create_album(name: str, artist: Artist, mbid: str = None) -> Album:
+def get_or_create_album(
+    name: str, artist: Artist, mbid: str = None
+) -> Optional[Album]:
     album = None
-    album_created = False
+    album_dict = lookup_album_dict_from_mb(name, artist_name=artist.name)
-    albums = Album.objects.filter(name__iexact=name)
-    if albums.count() == 1:
-        album = albums.first()
-    else:
-        for potential_album in albums:
-            if artist in album.artist_set.all():
-                album = potential_album
-    if not album:
-        album_created = True
-        album = Album.objects.create(name=name, musicbrainz_id=mbid)
-        album.save()
-        album.artists.add(artist)
 
-    if album_created or not mbid:
+    name = name or album_dict["title"]
-        album_dict = lookup_album_dict_from_mb(
+    album = Album.objects.filter(artists__in=[artist], name=name).first()
-            album.name, artist_name=artist.name
+
+    if not album and name:
+        mbid = mbid or album_dict["mb_id"]
+        album, album_created = Album.objects.get_or_create(
+            name=name, musicbrainz_id=mbid
         )
-        album.year = album_dict["year"]
+        if album_created:
-        album.musicbrainz_id = album_dict["mb_id"]
+            album.year = album_dict["year"]
-        album.musicbrainz_releasegroup_id = album_dict["mb_group_id"]
+            album.musicbrainz_releasegroup_id = album_dict["mb_group_id"]
-        album.musicbrainz_albumartist_id = artist.musicbrainz_id
+            album.musicbrainz_albumartist_id = artist.musicbrainz_id
-        album.save(
+            album.save(
-            update_fields=[
+                update_fields=[
-                "year",
+                    "musicbrainz_id",
-                "musicbrainz_id",
+                    "year",
-                "musicbrainz_releasegroup_id",
+                    "musicbrainz_releasegroup_id",
-                "musicbrainz_albumartist_id",
+                    "musicbrainz_albumartist_id",
-            ]
+                ]
-        )
+            )
-        album.artists.add(artist)
+            album.artists.add(artist)
-        album.fetch_artwork()
+            album.fix_album_artist()
+            album.fetch_artwork()
+            album.scrape_allmusic()
+
+    if not album:
+        logger.warn(f"No album found for {name} and {mbid}")
+
+    album.fix_album_artist()
     return album
 
 
 def get_or_create_track(
     title: str,
-    mbid: str,
     artist: Artist,
     album: Album,
-    run_time=None,
+    mbid: str = None,
-    run_time_ticks=None,
+    run_time_seconds=None,
 ) -> Track:
     track = None
-    if mbid:
+    if not mbid:
-        track = Track.objects.filter(
+        mbid = lookup_track_from_mb(
-            musicbrainz_id=mbid,
+            title,
-        ).first()
+            artist.musicbrainz_id,
-    if not track:
+            album.musicbrainz_id,
-        track = Track.objects.filter(
+        )["id"]
-            title=title, artist=artist, album=album
+
-        ).first()
+    track = Track.objects.filter(musicbrainz_id=mbid).first()
 
-    # TODO Can we look up mbid for tracks?
     if not track:
         track = Track.objects.create(
             title=title,
             artist=artist,
             album=album,
             musicbrainz_id=mbid,
-            run_time=run_time,
+            run_time_seconds=run_time_seconds,
-            run_time_ticks=run_time_ticks,
         )
 
     return track
+
+
+def get_or_create_various_artists():
+    artist = Artist.objects.filter(name="Various Artists").first()
+    if not artist:
+        artist = Artist.objects.create(**VARIOUS_ARTIST_DICT)
+        logger.info("Created Various Artists placeholder")
+    return artist

vrobbler/apps/music/views.py

@@ -1,10 +1,13 @@
+from django.db.models import Count
 from django.views import generic
-from music.models import Track, Artist, Album
+from music.models import Album, Artist, Track
+from scrobbles.models import ChartRecord
 from scrobbles.stats import get_scrobble_count_qs
 
 
 class TrackListView(generic.ListView):
     model = Track
+    paginate_by = 200
 
     def get_queryset(self):
         return get_scrobble_count_qs(user=self.request.user).order_by(
@@ -14,20 +17,78 @@ class TrackListView(generic.ListView):
 
 class TrackDetailView(generic.DetailView):
     model = Track
-    slug_field = 'uuid'
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        context_data["charts"] = ChartRecord.objects.filter(
+            track=self.object, rank__in=[1, 2, 3]
+        )
+        return context_data
 
 
 class ArtistListView(generic.ListView):
     model = Artist
+    paginate_by = 100
 
     def get_queryset(self):
-        return super().get_queryset().order_by("name")
+        return (
+            super()
+            .get_queryset()
+            .annotate(scrobble_count=Count("track__scrobble"))
+            .order_by("-scrobble_count")
+        )
+
+    def get_context_data(self, *, object_list=None, **kwargs):
+        context_data = super().get_context_data(
+            object_list=object_list, **kwargs
+        )
+        context_data["view"] = self.request.GET.get("view")
+        return context_data
 
 
 class ArtistDetailView(generic.DetailView):
     model = Artist
-    slug_field = 'uuid'
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        artist = context_data["object"]
+        rank = 1
+        tracks_ranked = []
+        scrobbles = artist.tracks.first().scrobble_count
+        for track in artist.tracks:
+            if scrobbles > track.scrobble_count:
+                rank += 1
+            tracks_ranked.append((rank, track))
+            scrobbles = track.scrobble_count
+
+        context_data["tracks_ranked"] = tracks_ranked
+        context_data["charts"] = ChartRecord.objects.filter(
+            artist=self.object, rank__in=[1, 2, 3]
+        )
+        return context_data
 
 
 class AlbumListView(generic.ListView):
     model = Album
+
+    def get_queryset(self):
+        return (
+            super()
+            .get_queryset()
+            .annotate(scrobble_count=Count("track__scrobble"))
+            .order_by("-scrobble_count")
+        )
+
+
+class AlbumDetailView(generic.DetailView):
+    model = Album
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        # context_data['charts'] = ChartRecord.objects.filter(
+        #    track__album=self.object, rank__in=[1, 2, 3]
+        # )
+        return context_data

vrobbler/apps/podcasts/admin.py

@@ -27,7 +27,6 @@ class EpisodeAdmin(admin.ModelAdmin):
     list_display = (
         "title",
         "podcast",
-        "run_time",
     )
     list_filter = ("podcast",)
     ordering = ("-created",)

vrobbler/apps/podcasts/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class PodcastsConfig(AppConfig):
-    name = 'podcasts'
+    name = "podcasts"

vrobbler/apps/podcasts/migrations/0006_alter_episode_run_time.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:01
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0005_alter_episode_options_alter_episode_title"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="episode",
+            name="run_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0007_rename_run_time_episode_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-12 01:21
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0006_alter_episode_run_time"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="episode",
+            old_name="run_time",
+            new_name="run_time_seconds",
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0008_episode_genre.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-03-14 22:27
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0033_genre_objectwithgenres"),
+        ("podcasts", "0007_rename_run_time_episode_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="episode",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0009_podcast_cover.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.5 on 2023-03-22 22:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0008_episode_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="podcast",
+            name="cover",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="pdocasts/covers/"
+            ),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0010_remove_podcast_cover_podcast_cover_image_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.1.5 on 2023-03-23 02:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0009_podcast_cover"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="podcast",
+            name="cover",
+        ),
+        migrations.AddField(
+            model_name="podcast",
+            name="cover_image",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="podcasts/covers/"
+            ),
+        ),
+        migrations.AddField(
+            model_name="podcast",
+            name="description",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/migrations/0011_rename_url_podcast_feed_url_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-23 04:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("podcasts", "0010_remove_podcast_cover_podcast_cover_image_and_more"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="podcast",
+            old_name="url",
+            new_name="feed_url",
+        ),
+        migrations.AddField(
+            model_name="podcast",
+            name="google_podcasts_url",
+            field=models.URLField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/podcasts/models.py

@@ -2,11 +2,15 @@ import logging
 from typing import Dict, Optional
 from uuid import uuid4
 
+import requests
+from django.apps import apps
 from django.conf import settings
+from django.core.files.base import ContentFile
 from django.db import models
+from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
-
+from podcasts.scrapers import scrape_data_from_google_podcasts
 from scrobbles.mixins import ScrobblableMixin
 
 logger = logging.getLogger(__name__)
@@ -27,15 +31,53 @@ class Podcast(TimeStampedModel):
     producer = models.ForeignKey(
         Producer, on_delete=models.DO_NOTHING, **BNULL
     )
+    description = models.TextField(**BNULL)
     active = models.BooleanField(default=True)
-    url = models.URLField(**BNULL)
+    feed_url = models.URLField(**BNULL)
+    google_podcasts_url = models.URLField(**BNULL)
+    cover_image = models.ImageField(upload_to="podcasts/covers/", **BNULL)
 
     def __str__(self):
         return f"{self.name}"
 
+    def get_absolute_url(self):
+        return reverse("podcasts:podcast_detail", kwargs={"slug": self.uuid})
+
+    def scrobbles(self, user):
+        Scrobble = apps.get_model("scrobbles", "Scrobble")
+        return Scrobble.objects.filter(
+            user=user, podcast_episode__podcast=self
+        ).order_by("-timestamp")
+
+    def scrape_google_podcasts(self, force=False):
+        podcast_dict = {}
+        if not self.cover_image or force:
+            podcast_dict = scrape_data_from_google_podcasts(self.name)
+            if podcast_dict:
+                if not self.producer:
+                    self.producer, created = Producer.objects.get_or_create(
+                        name=podcast_dict["producer"]
+                    )
+                self.description = podcast_dict.get("description")
+                self.google_podcasts_url = podcast_dict.get("google_url")
+                self.save(
+                    update_fields=[
+                        "description",
+                        "producer",
+                        "google_podcasts_url",
+                    ]
+                )
+
+        cover_url = podcast_dict.get("image_url")
+        if (not self.cover_image or force) and cover_url:
+            r = requests.get(cover_url)
+            if r.status_code == 200:
+                fname = f"{self.name}_{self.uuid}.jpg"
+                self.cover_image.save(fname, ContentFile(r.content), save=True)
+
 
 class Episode(ScrobblableMixin):
-    COMPLETION_PERCENT = getattr(settings, 'PODCAST_COMPLETION_PERCENT', 90)
+    COMPLETION_PERCENT = getattr(settings, "PODCAST_COMPLETION_PERCENT", 90)
 
     podcast = models.ForeignKey(Podcast, on_delete=models.DO_NOTHING)
     number = models.IntegerField(**BNULL)
@@ -45,6 +87,21 @@ class Episode(ScrobblableMixin):
     def __str__(self):
         return f"{self.title}"
 
+    @property
+    def subtitle(self):
+        return self.podcast
+
+    @property
+    def info_link(self):
+        return ""
+
+    @property
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.podcast.cover_image:
+            url = self.podcast.cover_image.url
+        return url
+
     @classmethod
     def find_or_create(
         cls, podcast_dict: Dict, producer_dict: Dict, episode_dict: Dict
@@ -53,12 +110,12 @@ class Episode(ScrobblableMixin):
         producer before saving the epsiode so it can be scrobbled.
 
         """
-        if not podcast_dict.get('name'):
+        if not podcast_dict.get("name"):
             logger.warning(f"No name from source for podcast, not scrobbling")
             return
 
         producer = None
-        if producer_dict.get('name'):
+        if producer_dict.get("name"):
             producer, producer_created = Producer.objects.get_or_create(
                 **producer_dict
             )
@@ -77,7 +134,7 @@ class Episode(ScrobblableMixin):
         else:
             logger.debug(f"Found podcast {podcast}")
 
-        episode_dict['podcast_id'] = podcast.id
+        episode_dict["podcast_id"] = podcast.id
 
         episode, created = cls.objects.get_or_create(**episode_dict)
         if created:

vrobbler/apps/podcasts/scrapers.py

@@ -0,0 +1,91 @@
+import urllib
+from typing import Optional
+from bs4 import BeautifulSoup
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+PODCAST_SEARCH_URL = "https://podcasts.google.com/search/{query}"
+
+
+def _strip_and_clean(text):
+    return text.replace("\n", " ").rstrip().lstrip()
+
+
+def _build_google_url(url):
+    return url.replace("./", "https://podcasts.google.com/")
+
+    return
+
+
+def _get_title_from_soup(soup) -> Optional[int]:
+    title = None
+    try:
+        potential_title = soup.find("div", class_="FyxyKd")
+        if potential_title:
+            title = _strip_and_clean(potential_title.get_text())
+    except ValueError:
+        pass
+    return title
+
+
+def _get_url_from_soup(soup) -> Optional[int]:
+    url = None
+    try:
+        url_tag = soup.find("a", class_="yXo2Qc")
+        if url_tag:
+            url = _build_google_url(url_tag.get("href"))
+    except ValueError:
+        pass
+    return url
+
+
+def _get_producer_from_soup(soup) -> str:
+    pub = ""
+    try:
+        potential_pub = soup.find("div", class_="J3Ov7d")
+        if potential_pub:
+            pub = _strip_and_clean(potential_pub.get_text())
+    except ValueError:
+        pass
+    return pub
+
+
+def _get_description_from_soup(soup) -> str:
+    desc = ""
+    try:
+        potential_desc = soup.find("div", class_="yuTZxb")
+        if potential_desc:
+            desc = _strip_and_clean(potential_desc.get_text())
+    except ValueError:
+        pass
+    return desc
+
+
+def _get_img_url_from_soup(soup) -> str:
+    url = ""
+    try:
+        img_tag = soup.find("img", class_="BhVIWc")
+        try:
+            url = img_tag["src"]
+        except IndexError:
+            pass
+    except ValueError:
+        pass
+    return url
+
+
+def scrape_data_from_google_podcasts(title) -> dict:
+    data_dict = {}
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    url = PODCAST_SEARCH_URL.format(query=title)
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "html.parser")
+        data_dict["title"] = _get_title_from_soup(soup)
+        data_dict["description"] = _get_description_from_soup(soup)
+        data_dict["producer"] = _get_producer_from_soup(soup)
+        data_dict["google_url"] = _get_url_from_soup(soup)
+        data_dict["image_url"] = _get_img_url_from_soup(soup)
+    return data_dict

vrobbler/apps/podcasts/urls.py

@@ -0,0 +1,14 @@
+from django.urls import path
+from podcasts import views
+
+app_name = "podcasts"
+
+
+urlpatterns = [
+    path("podcasts/", views.PodcastListView.as_view(), name="podcast_list"),
+    path(
+        "podcasts/<slug:slug>/",
+        views.PodcastDetailView.as_view(),
+        name="podcast_detail",
+    ),
+]

vrobbler/apps/podcasts/views.py

@@ -0,0 +1,18 @@
+from django.views import generic
+from podcasts.models import Podcast
+
+
+class PodcastListView(generic.ListView):
+    model = Podcast
+    paginate_by = 20
+
+
+class PodcastDetailView(generic.DetailView):
+    model = Podcast
+    slug_field = "uuid"
+
+    def get_context_data(self, **kwargs):
+        user = self.request.user
+        context_data = super().get_context_data(**kwargs)
+        context_data["scrobbles"] = self.object.scrobbles(user)
+        return context_data

vrobbler/apps/profiles/admin.py

@@ -7,3 +7,8 @@ from profiles.models import UserProfile
 class UserProfileAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     ordering = ("-created",)
+    exclude = (
+        "twitch_token",
+        "twitch_client_secret",
+        "lastfm_password",
+    )

vrobbler/apps/profiles/api/__init__.py

@@ -0,0 +1 @@
+#!/usr/bin/env python3

vrobbler/apps/profiles/api/serializers.py

@@ -0,0 +1,18 @@
+from django.contrib.auth import get_user_model
+from rest_framework import serializers
+
+User = get_user_model()
+
+from profiles.models import UserProfile
+
+
+class UserSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = User
+        exclude = ("password",)
+
+
+class UserProfileSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = UserProfile
+        exclude = ("lastfm_password",)

vrobbler/apps/profiles/api/views.py

@@ -0,0 +1,28 @@
+from django.contrib.auth import get_user_model
+
+from rest_framework import permissions, viewsets
+
+from profiles.api.serializers import UserSerializer, UserProfileSerializer
+from profiles.models import UserProfile
+
+User = get_user_model()
+
+
+class UserViewSet(viewsets.ModelViewSet):
+    """
+    API endpoint that allows users to be viewed or edited.
+    """
+
+    queryset = User.objects.all().order_by("-date_joined")
+    serializer_class = UserSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class UserProfileViewSet(viewsets.ModelViewSet):
+    """
+    API endpoint that allows users to be viewed or edited.
+    """
+
+    queryset = UserProfile.objects.all().order_by("-created")
+    serializer_class = UserProfileSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/profiles/migrations/0003_userprofile_twitch_client_id_and_more.py

@@ -0,0 +1,613 @@
+# Generated by Django 4.1.5 on 2023-03-05 00:05
+
+from django.db import migrations, models
+import encrypted_field.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0002_userprofile_lastfm_password_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="twitch_client_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="userprofile",
+            name="twitch_client_secret",
+            field=encrypted_field.fields.EncryptedField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="userprofile",
+            name="timezone",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("Pacific/Midway", "(GMT-1100) Pacific/Midway"),
+                    ("Pacific/Niue", "(GMT-1100) Pacific/Niue"),
+                    ("Pacific/Pago_Pago", "(GMT-1100) Pacific/Pago_Pago"),
+                    ("America/Adak", "(GMT-1000) America/Adak"),
+                    ("Pacific/Honolulu", "(GMT-1000) Pacific/Honolulu"),
+                    ("Pacific/Rarotonga", "(GMT-1000) Pacific/Rarotonga"),
+                    ("Pacific/Tahiti", "(GMT-1000) Pacific/Tahiti"),
+                    ("US/Hawaii", "(GMT-1000) US/Hawaii"),
+                    ("Pacific/Marquesas", "(GMT-0930) Pacific/Marquesas"),
+                    ("America/Anchorage", "(GMT-0900) America/Anchorage"),
+                    ("America/Juneau", "(GMT-0900) America/Juneau"),
+                    ("America/Metlakatla", "(GMT-0900) America/Metlakatla"),
+                    ("America/Nome", "(GMT-0900) America/Nome"),
+                    ("America/Sitka", "(GMT-0900) America/Sitka"),
+                    ("America/Yakutat", "(GMT-0900) America/Yakutat"),
+                    ("Pacific/Gambier", "(GMT-0900) Pacific/Gambier"),
+                    ("US/Alaska", "(GMT-0900) US/Alaska"),
+                    ("America/Los_Angeles", "(GMT-0800) America/Los_Angeles"),
+                    ("America/Tijuana", "(GMT-0800) America/Tijuana"),
+                    ("America/Vancouver", "(GMT-0800) America/Vancouver"),
+                    ("Canada/Pacific", "(GMT-0800) Canada/Pacific"),
+                    ("Pacific/Pitcairn", "(GMT-0800) Pacific/Pitcairn"),
+                    ("US/Pacific", "(GMT-0800) US/Pacific"),
+                    ("America/Boise", "(GMT-0700) America/Boise"),
+                    (
+                        "America/Cambridge_Bay",
+                        "(GMT-0700) America/Cambridge_Bay",
+                    ),
+                    (
+                        "America/Ciudad_Juarez",
+                        "(GMT-0700) America/Ciudad_Juarez",
+                    ),
+                    ("America/Creston", "(GMT-0700) America/Creston"),
+                    ("America/Dawson", "(GMT-0700) America/Dawson"),
+                    (
+                        "America/Dawson_Creek",
+                        "(GMT-0700) America/Dawson_Creek",
+                    ),
+                    ("America/Denver", "(GMT-0700) America/Denver"),
+                    ("America/Edmonton", "(GMT-0700) America/Edmonton"),
+                    ("America/Fort_Nelson", "(GMT-0700) America/Fort_Nelson"),
+                    ("America/Hermosillo", "(GMT-0700) America/Hermosillo"),
+                    ("America/Inuvik", "(GMT-0700) America/Inuvik"),
+                    ("America/Mazatlan", "(GMT-0700) America/Mazatlan"),
+                    ("America/Phoenix", "(GMT-0700) America/Phoenix"),
+                    ("America/Whitehorse", "(GMT-0700) America/Whitehorse"),
+                    ("America/Yellowknife", "(GMT-0700) America/Yellowknife"),
+                    ("Canada/Mountain", "(GMT-0700) Canada/Mountain"),
+                    ("US/Arizona", "(GMT-0700) US/Arizona"),
+                    ("US/Mountain", "(GMT-0700) US/Mountain"),
+                    (
+                        "America/Bahia_Banderas",
+                        "(GMT-0600) America/Bahia_Banderas",
+                    ),
+                    ("America/Belize", "(GMT-0600) America/Belize"),
+                    ("America/Chicago", "(GMT-0600) America/Chicago"),
+                    ("America/Chihuahua", "(GMT-0600) America/Chihuahua"),
+                    ("America/Costa_Rica", "(GMT-0600) America/Costa_Rica"),
+                    ("America/El_Salvador", "(GMT-0600) America/El_Salvador"),
+                    ("America/Guatemala", "(GMT-0600) America/Guatemala"),
+                    (
+                        "America/Indiana/Knox",
+                        "(GMT-0600) America/Indiana/Knox",
+                    ),
+                    (
+                        "America/Indiana/Tell_City",
+                        "(GMT-0600) America/Indiana/Tell_City",
+                    ),
+                    ("America/Managua", "(GMT-0600) America/Managua"),
+                    ("America/Matamoros", "(GMT-0600) America/Matamoros"),
+                    ("America/Menominee", "(GMT-0600) America/Menominee"),
+                    ("America/Merida", "(GMT-0600) America/Merida"),
+                    ("America/Mexico_City", "(GMT-0600) America/Mexico_City"),
+                    ("America/Monterrey", "(GMT-0600) America/Monterrey"),
+                    (
+                        "America/North_Dakota/Beulah",
+                        "(GMT-0600) America/North_Dakota/Beulah",
+                    ),
+                    (
+                        "America/North_Dakota/Center",
+                        "(GMT-0600) America/North_Dakota/Center",
+                    ),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "(GMT-0600) America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Ojinaga", "(GMT-0600) America/Ojinaga"),
+                    (
+                        "America/Rankin_Inlet",
+                        "(GMT-0600) America/Rankin_Inlet",
+                    ),
+                    ("America/Regina", "(GMT-0600) America/Regina"),
+                    ("America/Resolute", "(GMT-0600) America/Resolute"),
+                    (
+                        "America/Swift_Current",
+                        "(GMT-0600) America/Swift_Current",
+                    ),
+                    ("America/Tegucigalpa", "(GMT-0600) America/Tegucigalpa"),
+                    ("America/Winnipeg", "(GMT-0600) America/Winnipeg"),
+                    ("Canada/Central", "(GMT-0600) Canada/Central"),
+                    ("Pacific/Galapagos", "(GMT-0600) Pacific/Galapagos"),
+                    ("US/Central", "(GMT-0600) US/Central"),
+                    ("America/Atikokan", "(GMT-0500) America/Atikokan"),
+                    ("America/Bogota", "(GMT-0500) America/Bogota"),
+                    ("America/Cancun", "(GMT-0500) America/Cancun"),
+                    ("America/Cayman", "(GMT-0500) America/Cayman"),
+                    ("America/Detroit", "(GMT-0500) America/Detroit"),
+                    ("America/Eirunepe", "(GMT-0500) America/Eirunepe"),
+                    ("America/Grand_Turk", "(GMT-0500) America/Grand_Turk"),
+                    ("America/Guayaquil", "(GMT-0500) America/Guayaquil"),
+                    ("America/Havana", "(GMT-0500) America/Havana"),
+                    (
+                        "America/Indiana/Indianapolis",
+                        "(GMT-0500) America/Indiana/Indianapolis",
+                    ),
+                    (
+                        "America/Indiana/Marengo",
+                        "(GMT-0500) America/Indiana/Marengo",
+                    ),
+                    (
+                        "America/Indiana/Petersburg",
+                        "(GMT-0500) America/Indiana/Petersburg",
+                    ),
+                    (
+                        "America/Indiana/Vevay",
+                        "(GMT-0500) America/Indiana/Vevay",
+                    ),
+                    (
+                        "America/Indiana/Vincennes",
+                        "(GMT-0500) America/Indiana/Vincennes",
+                    ),
+                    (
+                        "America/Indiana/Winamac",
+                        "(GMT-0500) America/Indiana/Winamac",
+                    ),
+                    ("America/Iqaluit", "(GMT-0500) America/Iqaluit"),
+                    ("America/Jamaica", "(GMT-0500) America/Jamaica"),
+                    (
+                        "America/Kentucky/Louisville",
+                        "(GMT-0500) America/Kentucky/Louisville",
+                    ),
+                    (
+                        "America/Kentucky/Monticello",
+                        "(GMT-0500) America/Kentucky/Monticello",
+                    ),
+                    ("America/Lima", "(GMT-0500) America/Lima"),
+                    ("America/Nassau", "(GMT-0500) America/Nassau"),
+                    ("America/New_York", "(GMT-0500) America/New_York"),
+                    ("America/Panama", "(GMT-0500) America/Panama"),
+                    (
+                        "America/Port-au-Prince",
+                        "(GMT-0500) America/Port-au-Prince",
+                    ),
+                    ("America/Rio_Branco", "(GMT-0500) America/Rio_Branco"),
+                    ("America/Toronto", "(GMT-0500) America/Toronto"),
+                    ("Canada/Eastern", "(GMT-0500) Canada/Eastern"),
+                    ("Pacific/Easter", "(GMT-0500) Pacific/Easter"),
+                    ("US/Eastern", "(GMT-0500) US/Eastern"),
+                    ("America/Anguilla", "(GMT-0400) America/Anguilla"),
+                    ("America/Antigua", "(GMT-0400) America/Antigua"),
+                    ("America/Aruba", "(GMT-0400) America/Aruba"),
+                    ("America/Barbados", "(GMT-0400) America/Barbados"),
+                    (
+                        "America/Blanc-Sablon",
+                        "(GMT-0400) America/Blanc-Sablon",
+                    ),
+                    ("America/Boa_Vista", "(GMT-0400) America/Boa_Vista"),
+                    (
+                        "America/Campo_Grande",
+                        "(GMT-0400) America/Campo_Grande",
+                    ),
+                    ("America/Caracas", "(GMT-0400) America/Caracas"),
+                    ("America/Cuiaba", "(GMT-0400) America/Cuiaba"),
+                    ("America/Curacao", "(GMT-0400) America/Curacao"),
+                    ("America/Dominica", "(GMT-0400) America/Dominica"),
+                    ("America/Glace_Bay", "(GMT-0400) America/Glace_Bay"),
+                    ("America/Goose_Bay", "(GMT-0400) America/Goose_Bay"),
+                    ("America/Grenada", "(GMT-0400) America/Grenada"),
+                    ("America/Guadeloupe", "(GMT-0400) America/Guadeloupe"),
+                    ("America/Guyana", "(GMT-0400) America/Guyana"),
+                    ("America/Halifax", "(GMT-0400) America/Halifax"),
+                    ("America/Kralendijk", "(GMT-0400) America/Kralendijk"),
+                    ("America/La_Paz", "(GMT-0400) America/La_Paz"),
+                    (
+                        "America/Lower_Princes",
+                        "(GMT-0400) America/Lower_Princes",
+                    ),
+                    ("America/Manaus", "(GMT-0400) America/Manaus"),
+                    ("America/Marigot", "(GMT-0400) America/Marigot"),
+                    ("America/Martinique", "(GMT-0400) America/Martinique"),
+                    ("America/Moncton", "(GMT-0400) America/Moncton"),
+                    ("America/Montserrat", "(GMT-0400) America/Montserrat"),
+                    (
+                        "America/Port_of_Spain",
+                        "(GMT-0400) America/Port_of_Spain",
+                    ),
+                    ("America/Porto_Velho", "(GMT-0400) America/Porto_Velho"),
+                    ("America/Puerto_Rico", "(GMT-0400) America/Puerto_Rico"),
+                    (
+                        "America/Santo_Domingo",
+                        "(GMT-0400) America/Santo_Domingo",
+                    ),
+                    (
+                        "America/St_Barthelemy",
+                        "(GMT-0400) America/St_Barthelemy",
+                    ),
+                    ("America/St_Kitts", "(GMT-0400) America/St_Kitts"),
+                    ("America/St_Lucia", "(GMT-0400) America/St_Lucia"),
+                    ("America/St_Thomas", "(GMT-0400) America/St_Thomas"),
+                    ("America/St_Vincent", "(GMT-0400) America/St_Vincent"),
+                    ("America/Thule", "(GMT-0400) America/Thule"),
+                    ("America/Tortola", "(GMT-0400) America/Tortola"),
+                    ("Atlantic/Bermuda", "(GMT-0400) Atlantic/Bermuda"),
+                    ("Canada/Atlantic", "(GMT-0400) Canada/Atlantic"),
+                    ("America/St_Johns", "(GMT-0330) America/St_Johns"),
+                    ("Canada/Newfoundland", "(GMT-0330) Canada/Newfoundland"),
+                    ("America/Araguaina", "(GMT-0300) America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "(GMT-0300) America/Argentina/Buenos_Aires",
+                    ),
+                    (
+                        "America/Argentina/Catamarca",
+                        "(GMT-0300) America/Argentina/Catamarca",
+                    ),
+                    (
+                        "America/Argentina/Cordoba",
+                        "(GMT-0300) America/Argentina/Cordoba",
+                    ),
+                    (
+                        "America/Argentina/Jujuy",
+                        "(GMT-0300) America/Argentina/Jujuy",
+                    ),
+                    (
+                        "America/Argentina/La_Rioja",
+                        "(GMT-0300) America/Argentina/La_Rioja",
+                    ),
+                    (
+                        "America/Argentina/Mendoza",
+                        "(GMT-0300) America/Argentina/Mendoza",
+                    ),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "(GMT-0300) America/Argentina/Rio_Gallegos",
+                    ),
+                    (
+                        "America/Argentina/Salta",
+                        "(GMT-0300) America/Argentina/Salta",
+                    ),
+                    (
+                        "America/Argentina/San_Juan",
+                        "(GMT-0300) America/Argentina/San_Juan",
+                    ),
+                    (
+                        "America/Argentina/San_Luis",
+                        "(GMT-0300) America/Argentina/San_Luis",
+                    ),
+                    (
+                        "America/Argentina/Tucuman",
+                        "(GMT-0300) America/Argentina/Tucuman",
+                    ),
+                    (
+                        "America/Argentina/Ushuaia",
+                        "(GMT-0300) America/Argentina/Ushuaia",
+                    ),
+                    ("America/Asuncion", "(GMT-0300) America/Asuncion"),
+                    ("America/Bahia", "(GMT-0300) America/Bahia"),
+                    ("America/Belem", "(GMT-0300) America/Belem"),
+                    ("America/Cayenne", "(GMT-0300) America/Cayenne"),
+                    ("America/Fortaleza", "(GMT-0300) America/Fortaleza"),
+                    ("America/Maceio", "(GMT-0300) America/Maceio"),
+                    ("America/Miquelon", "(GMT-0300) America/Miquelon"),
+                    ("America/Montevideo", "(GMT-0300) America/Montevideo"),
+                    ("America/Nuuk", "(GMT-0300) America/Nuuk"),
+                    ("America/Paramaribo", "(GMT-0300) America/Paramaribo"),
+                    (
+                        "America/Punta_Arenas",
+                        "(GMT-0300) America/Punta_Arenas",
+                    ),
+                    ("America/Recife", "(GMT-0300) America/Recife"),
+                    ("America/Santarem", "(GMT-0300) America/Santarem"),
+                    ("America/Santiago", "(GMT-0300) America/Santiago"),
+                    ("America/Sao_Paulo", "(GMT-0300) America/Sao_Paulo"),
+                    ("Antarctica/Palmer", "(GMT-0300) Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "(GMT-0300) Antarctica/Rothera"),
+                    ("Atlantic/Stanley", "(GMT-0300) Atlantic/Stanley"),
+                    ("America/Noronha", "(GMT-0200) America/Noronha"),
+                    (
+                        "Atlantic/South_Georgia",
+                        "(GMT-0200) Atlantic/South_Georgia",
+                    ),
+                    (
+                        "America/Scoresbysund",
+                        "(GMT-0100) America/Scoresbysund",
+                    ),
+                    ("Atlantic/Azores", "(GMT-0100) Atlantic/Azores"),
+                    ("Atlantic/Cape_Verde", "(GMT-0100) Atlantic/Cape_Verde"),
+                    ("Africa/Abidjan", "(GMT+0000) Africa/Abidjan"),
+                    ("Africa/Accra", "(GMT+0000) Africa/Accra"),
+                    ("Africa/Bamako", "(GMT+0000) Africa/Bamako"),
+                    ("Africa/Banjul", "(GMT+0000) Africa/Banjul"),
+                    ("Africa/Bissau", "(GMT+0000) Africa/Bissau"),
+                    ("Africa/Conakry", "(GMT+0000) Africa/Conakry"),
+                    ("Africa/Dakar", "(GMT+0000) Africa/Dakar"),
+                    ("Africa/Freetown", "(GMT+0000) Africa/Freetown"),
+                    ("Africa/Lome", "(GMT+0000) Africa/Lome"),
+                    ("Africa/Monrovia", "(GMT+0000) Africa/Monrovia"),
+                    ("Africa/Nouakchott", "(GMT+0000) Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "(GMT+0000) Africa/Ouagadougou"),
+                    ("Africa/Sao_Tome", "(GMT+0000) Africa/Sao_Tome"),
+                    (
+                        "America/Danmarkshavn",
+                        "(GMT+0000) America/Danmarkshavn",
+                    ),
+                    ("Antarctica/Troll", "(GMT+0000) Antarctica/Troll"),
+                    ("Atlantic/Canary", "(GMT+0000) Atlantic/Canary"),
+                    ("Atlantic/Faroe", "(GMT+0000) Atlantic/Faroe"),
+                    ("Atlantic/Madeira", "(GMT+0000) Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "(GMT+0000) Atlantic/Reykjavik"),
+                    ("Atlantic/St_Helena", "(GMT+0000) Atlantic/St_Helena"),
+                    ("Europe/Dublin", "(GMT+0000) Europe/Dublin"),
+                    ("Europe/Guernsey", "(GMT+0000) Europe/Guernsey"),
+                    ("Europe/Isle_of_Man", "(GMT+0000) Europe/Isle_of_Man"),
+                    ("Europe/Jersey", "(GMT+0000) Europe/Jersey"),
+                    ("Europe/Lisbon", "(GMT+0000) Europe/Lisbon"),
+                    ("Europe/London", "(GMT+0000) Europe/London"),
+                    ("GMT", "(GMT+0000) GMT"),
+                    ("UTC", "(GMT+0000) UTC"),
+                    ("Africa/Algiers", "(GMT+0100) Africa/Algiers"),
+                    ("Africa/Bangui", "(GMT+0100) Africa/Bangui"),
+                    ("Africa/Brazzaville", "(GMT+0100) Africa/Brazzaville"),
+                    ("Africa/Casablanca", "(GMT+0100) Africa/Casablanca"),
+                    ("Africa/Ceuta", "(GMT+0100) Africa/Ceuta"),
+                    ("Africa/Douala", "(GMT+0100) Africa/Douala"),
+                    ("Africa/El_Aaiun", "(GMT+0100) Africa/El_Aaiun"),
+                    ("Africa/Kinshasa", "(GMT+0100) Africa/Kinshasa"),
+                    ("Africa/Lagos", "(GMT+0100) Africa/Lagos"),
+                    ("Africa/Libreville", "(GMT+0100) Africa/Libreville"),
+                    ("Africa/Luanda", "(GMT+0100) Africa/Luanda"),
+                    ("Africa/Malabo", "(GMT+0100) Africa/Malabo"),
+                    ("Africa/Ndjamena", "(GMT+0100) Africa/Ndjamena"),
+                    ("Africa/Niamey", "(GMT+0100) Africa/Niamey"),
+                    ("Africa/Porto-Novo", "(GMT+0100) Africa/Porto-Novo"),
+                    ("Africa/Tunis", "(GMT+0100) Africa/Tunis"),
+                    ("Arctic/Longyearbyen", "(GMT+0100) Arctic/Longyearbyen"),
+                    ("Europe/Amsterdam", "(GMT+0100) Europe/Amsterdam"),
+                    ("Europe/Andorra", "(GMT+0100) Europe/Andorra"),
+                    ("Europe/Belgrade", "(GMT+0100) Europe/Belgrade"),
+                    ("Europe/Berlin", "(GMT+0100) Europe/Berlin"),
+                    ("Europe/Bratislava", "(GMT+0100) Europe/Bratislava"),
+                    ("Europe/Brussels", "(GMT+0100) Europe/Brussels"),
+                    ("Europe/Budapest", "(GMT+0100) Europe/Budapest"),
+                    ("Europe/Busingen", "(GMT+0100) Europe/Busingen"),
+                    ("Europe/Copenhagen", "(GMT+0100) Europe/Copenhagen"),
+                    ("Europe/Gibraltar", "(GMT+0100) Europe/Gibraltar"),
+                    ("Europe/Ljubljana", "(GMT+0100) Europe/Ljubljana"),
+                    ("Europe/Luxembourg", "(GMT+0100) Europe/Luxembourg"),
+                    ("Europe/Madrid", "(GMT+0100) Europe/Madrid"),
+                    ("Europe/Malta", "(GMT+0100) Europe/Malta"),
+                    ("Europe/Monaco", "(GMT+0100) Europe/Monaco"),
+                    ("Europe/Oslo", "(GMT+0100) Europe/Oslo"),
+                    ("Europe/Paris", "(GMT+0100) Europe/Paris"),
+                    ("Europe/Podgorica", "(GMT+0100) Europe/Podgorica"),
+                    ("Europe/Prague", "(GMT+0100) Europe/Prague"),
+                    ("Europe/Rome", "(GMT+0100) Europe/Rome"),
+                    ("Europe/San_Marino", "(GMT+0100) Europe/San_Marino"),
+                    ("Europe/Sarajevo", "(GMT+0100) Europe/Sarajevo"),
+                    ("Europe/Skopje", "(GMT+0100) Europe/Skopje"),
+                    ("Europe/Stockholm", "(GMT+0100) Europe/Stockholm"),
+                    ("Europe/Tirane", "(GMT+0100) Europe/Tirane"),
+                    ("Europe/Vaduz", "(GMT+0100) Europe/Vaduz"),
+                    ("Europe/Vatican", "(GMT+0100) Europe/Vatican"),
+                    ("Europe/Vienna", "(GMT+0100) Europe/Vienna"),
+                    ("Europe/Warsaw", "(GMT+0100) Europe/Warsaw"),
+                    ("Europe/Zagreb", "(GMT+0100) Europe/Zagreb"),
+                    ("Europe/Zurich", "(GMT+0100) Europe/Zurich"),
+                    ("Africa/Blantyre", "(GMT+0200) Africa/Blantyre"),
+                    ("Africa/Bujumbura", "(GMT+0200) Africa/Bujumbura"),
+                    ("Africa/Cairo", "(GMT+0200) Africa/Cairo"),
+                    ("Africa/Gaborone", "(GMT+0200) Africa/Gaborone"),
+                    ("Africa/Harare", "(GMT+0200) Africa/Harare"),
+                    ("Africa/Johannesburg", "(GMT+0200) Africa/Johannesburg"),
+                    ("Africa/Juba", "(GMT+0200) Africa/Juba"),
+                    ("Africa/Khartoum", "(GMT+0200) Africa/Khartoum"),
+                    ("Africa/Kigali", "(GMT+0200) Africa/Kigali"),
+                    ("Africa/Lubumbashi", "(GMT+0200) Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "(GMT+0200) Africa/Lusaka"),
+                    ("Africa/Maputo", "(GMT+0200) Africa/Maputo"),
+                    ("Africa/Maseru", "(GMT+0200) Africa/Maseru"),
+                    ("Africa/Mbabane", "(GMT+0200) Africa/Mbabane"),
+                    ("Africa/Tripoli", "(GMT+0200) Africa/Tripoli"),
+                    ("Africa/Windhoek", "(GMT+0200) Africa/Windhoek"),
+                    ("Asia/Beirut", "(GMT+0200) Asia/Beirut"),
+                    ("Asia/Famagusta", "(GMT+0200) Asia/Famagusta"),
+                    ("Asia/Gaza", "(GMT+0200) Asia/Gaza"),
+                    ("Asia/Hebron", "(GMT+0200) Asia/Hebron"),
+                    ("Asia/Jerusalem", "(GMT+0200) Asia/Jerusalem"),
+                    ("Asia/Nicosia", "(GMT+0200) Asia/Nicosia"),
+                    ("Europe/Athens", "(GMT+0200) Europe/Athens"),
+                    ("Europe/Bucharest", "(GMT+0200) Europe/Bucharest"),
+                    ("Europe/Chisinau", "(GMT+0200) Europe/Chisinau"),
+                    ("Europe/Helsinki", "(GMT+0200) Europe/Helsinki"),
+                    ("Europe/Kaliningrad", "(GMT+0200) Europe/Kaliningrad"),
+                    ("Europe/Kyiv", "(GMT+0200) Europe/Kyiv"),
+                    ("Europe/Mariehamn", "(GMT+0200) Europe/Mariehamn"),
+                    ("Europe/Riga", "(GMT+0200) Europe/Riga"),
+                    ("Europe/Sofia", "(GMT+0200) Europe/Sofia"),
+                    ("Europe/Tallinn", "(GMT+0200) Europe/Tallinn"),
+                    ("Europe/Vilnius", "(GMT+0200) Europe/Vilnius"),
+                    ("Africa/Addis_Ababa", "(GMT+0300) Africa/Addis_Ababa"),
+                    ("Africa/Asmara", "(GMT+0300) Africa/Asmara"),
+                    (
+                        "Africa/Dar_es_Salaam",
+                        "(GMT+0300) Africa/Dar_es_Salaam",
+                    ),
+                    ("Africa/Djibouti", "(GMT+0300) Africa/Djibouti"),
+                    ("Africa/Kampala", "(GMT+0300) Africa/Kampala"),
+                    ("Africa/Mogadishu", "(GMT+0300) Africa/Mogadishu"),
+                    ("Africa/Nairobi", "(GMT+0300) Africa/Nairobi"),
+                    ("Antarctica/Syowa", "(GMT+0300) Antarctica/Syowa"),
+                    ("Asia/Aden", "(GMT+0300) Asia/Aden"),
+                    ("Asia/Amman", "(GMT+0300) Asia/Amman"),
+                    ("Asia/Baghdad", "(GMT+0300) Asia/Baghdad"),
+                    ("Asia/Bahrain", "(GMT+0300) Asia/Bahrain"),
+                    ("Asia/Damascus", "(GMT+0300) Asia/Damascus"),
+                    ("Asia/Kuwait", "(GMT+0300) Asia/Kuwait"),
+                    ("Asia/Qatar", "(GMT+0300) Asia/Qatar"),
+                    ("Asia/Riyadh", "(GMT+0300) Asia/Riyadh"),
+                    ("Europe/Istanbul", "(GMT+0300) Europe/Istanbul"),
+                    ("Europe/Kirov", "(GMT+0300) Europe/Kirov"),
+                    ("Europe/Minsk", "(GMT+0300) Europe/Minsk"),
+                    ("Europe/Moscow", "(GMT+0300) Europe/Moscow"),
+                    ("Europe/Simferopol", "(GMT+0300) Europe/Simferopol"),
+                    ("Europe/Volgograd", "(GMT+0300) Europe/Volgograd"),
+                    ("Indian/Antananarivo", "(GMT+0300) Indian/Antananarivo"),
+                    ("Indian/Comoro", "(GMT+0300) Indian/Comoro"),
+                    ("Indian/Mayotte", "(GMT+0300) Indian/Mayotte"),
+                    ("Asia/Tehran", "(GMT+0330) Asia/Tehran"),
+                    ("Asia/Baku", "(GMT+0400) Asia/Baku"),
+                    ("Asia/Dubai", "(GMT+0400) Asia/Dubai"),
+                    ("Asia/Muscat", "(GMT+0400) Asia/Muscat"),
+                    ("Asia/Tbilisi", "(GMT+0400) Asia/Tbilisi"),
+                    ("Asia/Yerevan", "(GMT+0400) Asia/Yerevan"),
+                    ("Europe/Astrakhan", "(GMT+0400) Europe/Astrakhan"),
+                    ("Europe/Samara", "(GMT+0400) Europe/Samara"),
+                    ("Europe/Saratov", "(GMT+0400) Europe/Saratov"),
+                    ("Europe/Ulyanovsk", "(GMT+0400) Europe/Ulyanovsk"),
+                    ("Indian/Mahe", "(GMT+0400) Indian/Mahe"),
+                    ("Indian/Mauritius", "(GMT+0400) Indian/Mauritius"),
+                    ("Indian/Reunion", "(GMT+0400) Indian/Reunion"),
+                    ("Asia/Kabul", "(GMT+0430) Asia/Kabul"),
+                    ("Antarctica/Mawson", "(GMT+0500) Antarctica/Mawson"),
+                    ("Asia/Aqtau", "(GMT+0500) Asia/Aqtau"),
+                    ("Asia/Aqtobe", "(GMT+0500) Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "(GMT+0500) Asia/Ashgabat"),
+                    ("Asia/Atyrau", "(GMT+0500) Asia/Atyrau"),
+                    ("Asia/Dushanbe", "(GMT+0500) Asia/Dushanbe"),
+                    ("Asia/Karachi", "(GMT+0500) Asia/Karachi"),
+                    ("Asia/Oral", "(GMT+0500) Asia/Oral"),
+                    ("Asia/Qyzylorda", "(GMT+0500) Asia/Qyzylorda"),
+                    ("Asia/Samarkand", "(GMT+0500) Asia/Samarkand"),
+                    ("Asia/Tashkent", "(GMT+0500) Asia/Tashkent"),
+                    ("Asia/Yekaterinburg", "(GMT+0500) Asia/Yekaterinburg"),
+                    ("Indian/Kerguelen", "(GMT+0500) Indian/Kerguelen"),
+                    ("Indian/Maldives", "(GMT+0500) Indian/Maldives"),
+                    ("Asia/Colombo", "(GMT+0530) Asia/Colombo"),
+                    ("Asia/Kolkata", "(GMT+0530) Asia/Kolkata"),
+                    ("Asia/Kathmandu", "(GMT+0545) Asia/Kathmandu"),
+                    ("Antarctica/Vostok", "(GMT+0600) Antarctica/Vostok"),
+                    ("Asia/Almaty", "(GMT+0600) Asia/Almaty"),
+                    ("Asia/Bishkek", "(GMT+0600) Asia/Bishkek"),
+                    ("Asia/Dhaka", "(GMT+0600) Asia/Dhaka"),
+                    ("Asia/Omsk", "(GMT+0600) Asia/Omsk"),
+                    ("Asia/Qostanay", "(GMT+0600) Asia/Qostanay"),
+                    ("Asia/Thimphu", "(GMT+0600) Asia/Thimphu"),
+                    ("Asia/Urumqi", "(GMT+0600) Asia/Urumqi"),
+                    ("Indian/Chagos", "(GMT+0600) Indian/Chagos"),
+                    ("Asia/Yangon", "(GMT+0630) Asia/Yangon"),
+                    ("Indian/Cocos", "(GMT+0630) Indian/Cocos"),
+                    ("Antarctica/Davis", "(GMT+0700) Antarctica/Davis"),
+                    ("Asia/Bangkok", "(GMT+0700) Asia/Bangkok"),
+                    ("Asia/Barnaul", "(GMT+0700) Asia/Barnaul"),
+                    ("Asia/Ho_Chi_Minh", "(GMT+0700) Asia/Ho_Chi_Minh"),
+                    ("Asia/Hovd", "(GMT+0700) Asia/Hovd"),
+                    ("Asia/Jakarta", "(GMT+0700) Asia/Jakarta"),
+                    ("Asia/Krasnoyarsk", "(GMT+0700) Asia/Krasnoyarsk"),
+                    ("Asia/Novokuznetsk", "(GMT+0700) Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "(GMT+0700) Asia/Novosibirsk"),
+                    ("Asia/Phnom_Penh", "(GMT+0700) Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "(GMT+0700) Asia/Pontianak"),
+                    ("Asia/Tomsk", "(GMT+0700) Asia/Tomsk"),
+                    ("Asia/Vientiane", "(GMT+0700) Asia/Vientiane"),
+                    ("Indian/Christmas", "(GMT+0700) Indian/Christmas"),
+                    ("Asia/Brunei", "(GMT+0800) Asia/Brunei"),
+                    ("Asia/Choibalsan", "(GMT+0800) Asia/Choibalsan"),
+                    ("Asia/Hong_Kong", "(GMT+0800) Asia/Hong_Kong"),
+                    ("Asia/Irkutsk", "(GMT+0800) Asia/Irkutsk"),
+                    ("Asia/Kuala_Lumpur", "(GMT+0800) Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "(GMT+0800) Asia/Kuching"),
+                    ("Asia/Macau", "(GMT+0800) Asia/Macau"),
+                    ("Asia/Makassar", "(GMT+0800) Asia/Makassar"),
+                    ("Asia/Manila", "(GMT+0800) Asia/Manila"),
+                    ("Asia/Shanghai", "(GMT+0800) Asia/Shanghai"),
+                    ("Asia/Singapore", "(GMT+0800) Asia/Singapore"),
+                    ("Asia/Taipei", "(GMT+0800) Asia/Taipei"),
+                    ("Asia/Ulaanbaatar", "(GMT+0800) Asia/Ulaanbaatar"),
+                    ("Australia/Perth", "(GMT+0800) Australia/Perth"),
+                    ("Australia/Eucla", "(GMT+0845) Australia/Eucla"),
+                    ("Asia/Chita", "(GMT+0900) Asia/Chita"),
+                    ("Asia/Dili", "(GMT+0900) Asia/Dili"),
+                    ("Asia/Jayapura", "(GMT+0900) Asia/Jayapura"),
+                    ("Asia/Khandyga", "(GMT+0900) Asia/Khandyga"),
+                    ("Asia/Pyongyang", "(GMT+0900) Asia/Pyongyang"),
+                    ("Asia/Seoul", "(GMT+0900) Asia/Seoul"),
+                    ("Asia/Tokyo", "(GMT+0900) Asia/Tokyo"),
+                    ("Asia/Yakutsk", "(GMT+0900) Asia/Yakutsk"),
+                    ("Pacific/Palau", "(GMT+0900) Pacific/Palau"),
+                    ("Australia/Darwin", "(GMT+0930) Australia/Darwin"),
+                    (
+                        "Antarctica/DumontDUrville",
+                        "(GMT+1000) Antarctica/DumontDUrville",
+                    ),
+                    ("Asia/Ust-Nera", "(GMT+1000) Asia/Ust-Nera"),
+                    ("Asia/Vladivostok", "(GMT+1000) Asia/Vladivostok"),
+                    ("Australia/Brisbane", "(GMT+1000) Australia/Brisbane"),
+                    ("Australia/Lindeman", "(GMT+1000) Australia/Lindeman"),
+                    ("Pacific/Chuuk", "(GMT+1000) Pacific/Chuuk"),
+                    ("Pacific/Guam", "(GMT+1000) Pacific/Guam"),
+                    (
+                        "Pacific/Port_Moresby",
+                        "(GMT+1000) Pacific/Port_Moresby",
+                    ),
+                    ("Pacific/Saipan", "(GMT+1000) Pacific/Saipan"),
+                    ("Australia/Adelaide", "(GMT+1030) Australia/Adelaide"),
+                    (
+                        "Australia/Broken_Hill",
+                        "(GMT+1030) Australia/Broken_Hill",
+                    ),
+                    ("Antarctica/Casey", "(GMT+1100) Antarctica/Casey"),
+                    (
+                        "Antarctica/Macquarie",
+                        "(GMT+1100) Antarctica/Macquarie",
+                    ),
+                    ("Asia/Magadan", "(GMT+1100) Asia/Magadan"),
+                    ("Asia/Sakhalin", "(GMT+1100) Asia/Sakhalin"),
+                    ("Asia/Srednekolymsk", "(GMT+1100) Asia/Srednekolymsk"),
+                    ("Australia/Hobart", "(GMT+1100) Australia/Hobart"),
+                    ("Australia/Lord_Howe", "(GMT+1100) Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "(GMT+1100) Australia/Melbourne"),
+                    ("Australia/Sydney", "(GMT+1100) Australia/Sydney"),
+                    (
+                        "Pacific/Bougainville",
+                        "(GMT+1100) Pacific/Bougainville",
+                    ),
+                    ("Pacific/Efate", "(GMT+1100) Pacific/Efate"),
+                    ("Pacific/Guadalcanal", "(GMT+1100) Pacific/Guadalcanal"),
+                    ("Pacific/Kosrae", "(GMT+1100) Pacific/Kosrae"),
+                    ("Pacific/Noumea", "(GMT+1100) Pacific/Noumea"),
+                    ("Pacific/Pohnpei", "(GMT+1100) Pacific/Pohnpei"),
+                    ("Asia/Anadyr", "(GMT+1200) Asia/Anadyr"),
+                    ("Asia/Kamchatka", "(GMT+1200) Asia/Kamchatka"),
+                    ("Pacific/Fiji", "(GMT+1200) Pacific/Fiji"),
+                    ("Pacific/Funafuti", "(GMT+1200) Pacific/Funafuti"),
+                    ("Pacific/Kwajalein", "(GMT+1200) Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "(GMT+1200) Pacific/Majuro"),
+                    ("Pacific/Nauru", "(GMT+1200) Pacific/Nauru"),
+                    ("Pacific/Norfolk", "(GMT+1200) Pacific/Norfolk"),
+                    ("Pacific/Tarawa", "(GMT+1200) Pacific/Tarawa"),
+                    ("Pacific/Wake", "(GMT+1200) Pacific/Wake"),
+                    ("Pacific/Wallis", "(GMT+1200) Pacific/Wallis"),
+                    ("Antarctica/McMurdo", "(GMT+1300) Antarctica/McMurdo"),
+                    ("Pacific/Apia", "(GMT+1300) Pacific/Apia"),
+                    ("Pacific/Auckland", "(GMT+1300) Pacific/Auckland"),
+                    ("Pacific/Fakaofo", "(GMT+1300) Pacific/Fakaofo"),
+                    ("Pacific/Kanton", "(GMT+1300) Pacific/Kanton"),
+                    ("Pacific/Tongatapu", "(GMT+1300) Pacific/Tongatapu"),
+                    ("Pacific/Chatham", "(GMT+1345) Pacific/Chatham"),
+                    ("Pacific/Kiritimati", "(GMT+1400) Pacific/Kiritimati"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

vrobbler/apps/profiles/migrations/0004_userprofile_twitch_token_and_more.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.5 on 2023-03-05 01:04
+
+from django.db import migrations, models
+import encrypted_field.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0003_userprofile_twitch_client_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="twitch_token",
+            field=encrypted_field.fields.EncryptedField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="userprofile",
+            name="twitch_token_expires",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/profiles/migrations/0005_remove_userprofile_twitch_client_id_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.1.5 on 2023-03-05 21:27
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0004_userprofile_twitch_token_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="userprofile",
+            name="twitch_client_id",
+        ),
+        migrations.RemoveField(
+            model_name="userprofile",
+            name="twitch_client_secret",
+        ),
+        migrations.RemoveField(
+            model_name="userprofile",
+            name="twitch_token",
+        ),
+        migrations.RemoveField(
+            model_name="userprofile",
+            name="twitch_token_expires",
+        ),
+    ]

vrobbler/apps/profiles/migrations/0006_alter_userprofile_timezone.py

@@ -0,0 +1,602 @@
+# Generated by Django 4.1.5 on 2023-03-13 04:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0005_remove_userprofile_twitch_client_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="userprofile",
+            name="timezone",
+            field=models.CharField(
+                blank=True,
+                choices=[
+                    ("Pacific/Midway", "(GMT-1100) Pacific/Midway"),
+                    ("Pacific/Niue", "(GMT-1100) Pacific/Niue"),
+                    ("Pacific/Pago_Pago", "(GMT-1100) Pacific/Pago_Pago"),
+                    ("Pacific/Honolulu", "(GMT-1000) Pacific/Honolulu"),
+                    ("Pacific/Rarotonga", "(GMT-1000) Pacific/Rarotonga"),
+                    ("Pacific/Tahiti", "(GMT-1000) Pacific/Tahiti"),
+                    ("US/Hawaii", "(GMT-1000) US/Hawaii"),
+                    ("Pacific/Marquesas", "(GMT-0930) Pacific/Marquesas"),
+                    ("America/Adak", "(GMT-0900) America/Adak"),
+                    ("Pacific/Gambier", "(GMT-0900) Pacific/Gambier"),
+                    ("America/Anchorage", "(GMT-0800) America/Anchorage"),
+                    ("America/Juneau", "(GMT-0800) America/Juneau"),
+                    ("America/Metlakatla", "(GMT-0800) America/Metlakatla"),
+                    ("America/Nome", "(GMT-0800) America/Nome"),
+                    ("America/Sitka", "(GMT-0800) America/Sitka"),
+                    ("America/Yakutat", "(GMT-0800) America/Yakutat"),
+                    ("Pacific/Pitcairn", "(GMT-0800) Pacific/Pitcairn"),
+                    ("US/Alaska", "(GMT-0800) US/Alaska"),
+                    ("America/Creston", "(GMT-0700) America/Creston"),
+                    ("America/Dawson", "(GMT-0700) America/Dawson"),
+                    (
+                        "America/Dawson_Creek",
+                        "(GMT-0700) America/Dawson_Creek",
+                    ),
+                    ("America/Fort_Nelson", "(GMT-0700) America/Fort_Nelson"),
+                    ("America/Hermosillo", "(GMT-0700) America/Hermosillo"),
+                    ("America/Los_Angeles", "(GMT-0700) America/Los_Angeles"),
+                    ("America/Mazatlan", "(GMT-0700) America/Mazatlan"),
+                    ("America/Phoenix", "(GMT-0700) America/Phoenix"),
+                    ("America/Tijuana", "(GMT-0700) America/Tijuana"),
+                    ("America/Vancouver", "(GMT-0700) America/Vancouver"),
+                    ("America/Whitehorse", "(GMT-0700) America/Whitehorse"),
+                    ("Canada/Pacific", "(GMT-0700) Canada/Pacific"),
+                    ("US/Arizona", "(GMT-0700) US/Arizona"),
+                    ("US/Pacific", "(GMT-0700) US/Pacific"),
+                    (
+                        "America/Bahia_Banderas",
+                        "(GMT-0600) America/Bahia_Banderas",
+                    ),
+                    ("America/Belize", "(GMT-0600) America/Belize"),
+                    ("America/Boise", "(GMT-0600) America/Boise"),
+                    (
+                        "America/Cambridge_Bay",
+                        "(GMT-0600) America/Cambridge_Bay",
+                    ),
+                    ("America/Chihuahua", "(GMT-0600) America/Chihuahua"),
+                    (
+                        "America/Ciudad_Juarez",
+                        "(GMT-0600) America/Ciudad_Juarez",
+                    ),
+                    ("America/Costa_Rica", "(GMT-0600) America/Costa_Rica"),
+                    ("America/Denver", "(GMT-0600) America/Denver"),
+                    ("America/Edmonton", "(GMT-0600) America/Edmonton"),
+                    ("America/El_Salvador", "(GMT-0600) America/El_Salvador"),
+                    ("America/Guatemala", "(GMT-0600) America/Guatemala"),
+                    ("America/Inuvik", "(GMT-0600) America/Inuvik"),
+                    ("America/Managua", "(GMT-0600) America/Managua"),
+                    ("America/Merida", "(GMT-0600) America/Merida"),
+                    ("America/Mexico_City", "(GMT-0600) America/Mexico_City"),
+                    ("America/Monterrey", "(GMT-0600) America/Monterrey"),
+                    ("America/Regina", "(GMT-0600) America/Regina"),
+                    (
+                        "America/Swift_Current",
+                        "(GMT-0600) America/Swift_Current",
+                    ),
+                    ("America/Tegucigalpa", "(GMT-0600) America/Tegucigalpa"),
+                    ("America/Yellowknife", "(GMT-0600) America/Yellowknife"),
+                    ("Canada/Mountain", "(GMT-0600) Canada/Mountain"),
+                    ("Pacific/Galapagos", "(GMT-0600) Pacific/Galapagos"),
+                    ("US/Mountain", "(GMT-0600) US/Mountain"),
+                    ("America/Atikokan", "(GMT-0500) America/Atikokan"),
+                    ("America/Bogota", "(GMT-0500) America/Bogota"),
+                    ("America/Cancun", "(GMT-0500) America/Cancun"),
+                    ("America/Cayman", "(GMT-0500) America/Cayman"),
+                    ("America/Chicago", "(GMT-0500) America/Chicago"),
+                    ("America/Eirunepe", "(GMT-0500) America/Eirunepe"),
+                    ("America/Guayaquil", "(GMT-0500) America/Guayaquil"),
+                    (
+                        "America/Indiana/Knox",
+                        "(GMT-0500) America/Indiana/Knox",
+                    ),
+                    (
+                        "America/Indiana/Tell_City",
+                        "(GMT-0500) America/Indiana/Tell_City",
+                    ),
+                    ("America/Jamaica", "(GMT-0500) America/Jamaica"),
+                    ("America/Lima", "(GMT-0500) America/Lima"),
+                    ("America/Matamoros", "(GMT-0500) America/Matamoros"),
+                    ("America/Menominee", "(GMT-0500) America/Menominee"),
+                    (
+                        "America/North_Dakota/Beulah",
+                        "(GMT-0500) America/North_Dakota/Beulah",
+                    ),
+                    (
+                        "America/North_Dakota/Center",
+                        "(GMT-0500) America/North_Dakota/Center",
+                    ),
+                    (
+                        "America/North_Dakota/New_Salem",
+                        "(GMT-0500) America/North_Dakota/New_Salem",
+                    ),
+                    ("America/Ojinaga", "(GMT-0500) America/Ojinaga"),
+                    ("America/Panama", "(GMT-0500) America/Panama"),
+                    (
+                        "America/Rankin_Inlet",
+                        "(GMT-0500) America/Rankin_Inlet",
+                    ),
+                    ("America/Resolute", "(GMT-0500) America/Resolute"),
+                    ("America/Rio_Branco", "(GMT-0500) America/Rio_Branco"),
+                    ("America/Winnipeg", "(GMT-0500) America/Winnipeg"),
+                    ("Canada/Central", "(GMT-0500) Canada/Central"),
+                    ("Pacific/Easter", "(GMT-0500) Pacific/Easter"),
+                    ("US/Central", "(GMT-0500) US/Central"),
+                    ("America/Anguilla", "(GMT-0400) America/Anguilla"),
+                    ("America/Antigua", "(GMT-0400) America/Antigua"),
+                    ("America/Aruba", "(GMT-0400) America/Aruba"),
+                    ("America/Barbados", "(GMT-0400) America/Barbados"),
+                    (
+                        "America/Blanc-Sablon",
+                        "(GMT-0400) America/Blanc-Sablon",
+                    ),
+                    ("America/Boa_Vista", "(GMT-0400) America/Boa_Vista"),
+                    (
+                        "America/Campo_Grande",
+                        "(GMT-0400) America/Campo_Grande",
+                    ),
+                    ("America/Caracas", "(GMT-0400) America/Caracas"),
+                    ("America/Cuiaba", "(GMT-0400) America/Cuiaba"),
+                    ("America/Curacao", "(GMT-0400) America/Curacao"),
+                    ("America/Detroit", "(GMT-0400) America/Detroit"),
+                    ("America/Dominica", "(GMT-0400) America/Dominica"),
+                    ("America/Grand_Turk", "(GMT-0400) America/Grand_Turk"),
+                    ("America/Grenada", "(GMT-0400) America/Grenada"),
+                    ("America/Guadeloupe", "(GMT-0400) America/Guadeloupe"),
+                    ("America/Guyana", "(GMT-0400) America/Guyana"),
+                    ("America/Havana", "(GMT-0400) America/Havana"),
+                    (
+                        "America/Indiana/Indianapolis",
+                        "(GMT-0400) America/Indiana/Indianapolis",
+                    ),
+                    (
+                        "America/Indiana/Marengo",
+                        "(GMT-0400) America/Indiana/Marengo",
+                    ),
+                    (
+                        "America/Indiana/Petersburg",
+                        "(GMT-0400) America/Indiana/Petersburg",
+                    ),
+                    (
+                        "America/Indiana/Vevay",
+                        "(GMT-0400) America/Indiana/Vevay",
+                    ),
+                    (
+                        "America/Indiana/Vincennes",
+                        "(GMT-0400) America/Indiana/Vincennes",
+                    ),
+                    (
+                        "America/Indiana/Winamac",
+                        "(GMT-0400) America/Indiana/Winamac",
+                    ),
+                    ("America/Iqaluit", "(GMT-0400) America/Iqaluit"),
+                    (
+                        "America/Kentucky/Louisville",
+                        "(GMT-0400) America/Kentucky/Louisville",
+                    ),
+                    (
+                        "America/Kentucky/Monticello",
+                        "(GMT-0400) America/Kentucky/Monticello",
+                    ),
+                    ("America/Kralendijk", "(GMT-0400) America/Kralendijk"),
+                    ("America/La_Paz", "(GMT-0400) America/La_Paz"),
+                    (
+                        "America/Lower_Princes",
+                        "(GMT-0400) America/Lower_Princes",
+                    ),
+                    ("America/Manaus", "(GMT-0400) America/Manaus"),
+                    ("America/Marigot", "(GMT-0400) America/Marigot"),
+                    ("America/Martinique", "(GMT-0400) America/Martinique"),
+                    ("America/Montserrat", "(GMT-0400) America/Montserrat"),
+                    ("America/Nassau", "(GMT-0400) America/Nassau"),
+                    ("America/New_York", "(GMT-0400) America/New_York"),
+                    (
+                        "America/Port-au-Prince",
+                        "(GMT-0400) America/Port-au-Prince",
+                    ),
+                    (
+                        "America/Port_of_Spain",
+                        "(GMT-0400) America/Port_of_Spain",
+                    ),
+                    ("America/Porto_Velho", "(GMT-0400) America/Porto_Velho"),
+                    ("America/Puerto_Rico", "(GMT-0400) America/Puerto_Rico"),
+                    (
+                        "America/Santo_Domingo",
+                        "(GMT-0400) America/Santo_Domingo",
+                    ),
+                    (
+                        "America/St_Barthelemy",
+                        "(GMT-0400) America/St_Barthelemy",
+                    ),
+                    ("America/St_Kitts", "(GMT-0400) America/St_Kitts"),
+                    ("America/St_Lucia", "(GMT-0400) America/St_Lucia"),
+                    ("America/St_Thomas", "(GMT-0400) America/St_Thomas"),
+                    ("America/St_Vincent", "(GMT-0400) America/St_Vincent"),
+                    ("America/Toronto", "(GMT-0400) America/Toronto"),
+                    ("America/Tortola", "(GMT-0400) America/Tortola"),
+                    ("Canada/Eastern", "(GMT-0400) Canada/Eastern"),
+                    ("US/Eastern", "(GMT-0400) US/Eastern"),
+                    ("America/Araguaina", "(GMT-0300) America/Araguaina"),
+                    (
+                        "America/Argentina/Buenos_Aires",
+                        "(GMT-0300) America/Argentina/Buenos_Aires",
+                    ),
+                    (
+                        "America/Argentina/Catamarca",
+                        "(GMT-0300) America/Argentina/Catamarca",
+                    ),
+                    (
+                        "America/Argentina/Cordoba",
+                        "(GMT-0300) America/Argentina/Cordoba",
+                    ),
+                    (
+                        "America/Argentina/Jujuy",
+                        "(GMT-0300) America/Argentina/Jujuy",
+                    ),
+                    (
+                        "America/Argentina/La_Rioja",
+                        "(GMT-0300) America/Argentina/La_Rioja",
+                    ),
+                    (
+                        "America/Argentina/Mendoza",
+                        "(GMT-0300) America/Argentina/Mendoza",
+                    ),
+                    (
+                        "America/Argentina/Rio_Gallegos",
+                        "(GMT-0300) America/Argentina/Rio_Gallegos",
+                    ),
+                    (
+                        "America/Argentina/Salta",
+                        "(GMT-0300) America/Argentina/Salta",
+                    ),
+                    (
+                        "America/Argentina/San_Juan",
+                        "(GMT-0300) America/Argentina/San_Juan",
+                    ),
+                    (
+                        "America/Argentina/San_Luis",
+                        "(GMT-0300) America/Argentina/San_Luis",
+                    ),
+                    (
+                        "America/Argentina/Tucuman",
+                        "(GMT-0300) America/Argentina/Tucuman",
+                    ),
+                    (
+                        "America/Argentina/Ushuaia",
+                        "(GMT-0300) America/Argentina/Ushuaia",
+                    ),
+                    ("America/Asuncion", "(GMT-0300) America/Asuncion"),
+                    ("America/Bahia", "(GMT-0300) America/Bahia"),
+                    ("America/Belem", "(GMT-0300) America/Belem"),
+                    ("America/Cayenne", "(GMT-0300) America/Cayenne"),
+                    ("America/Fortaleza", "(GMT-0300) America/Fortaleza"),
+                    ("America/Glace_Bay", "(GMT-0300) America/Glace_Bay"),
+                    ("America/Goose_Bay", "(GMT-0300) America/Goose_Bay"),
+                    ("America/Halifax", "(GMT-0300) America/Halifax"),
+                    ("America/Maceio", "(GMT-0300) America/Maceio"),
+                    ("America/Moncton", "(GMT-0300) America/Moncton"),
+                    ("America/Montevideo", "(GMT-0300) America/Montevideo"),
+                    ("America/Nuuk", "(GMT-0300) America/Nuuk"),
+                    ("America/Paramaribo", "(GMT-0300) America/Paramaribo"),
+                    (
+                        "America/Punta_Arenas",
+                        "(GMT-0300) America/Punta_Arenas",
+                    ),
+                    ("America/Recife", "(GMT-0300) America/Recife"),
+                    ("America/Santarem", "(GMT-0300) America/Santarem"),
+                    ("America/Santiago", "(GMT-0300) America/Santiago"),
+                    ("America/Sao_Paulo", "(GMT-0300) America/Sao_Paulo"),
+                    ("America/Thule", "(GMT-0300) America/Thule"),
+                    ("Antarctica/Palmer", "(GMT-0300) Antarctica/Palmer"),
+                    ("Antarctica/Rothera", "(GMT-0300) Antarctica/Rothera"),
+                    ("Atlantic/Bermuda", "(GMT-0300) Atlantic/Bermuda"),
+                    ("Atlantic/Stanley", "(GMT-0300) Atlantic/Stanley"),
+                    ("Canada/Atlantic", "(GMT-0300) Canada/Atlantic"),
+                    ("America/St_Johns", "(GMT-0230) America/St_Johns"),
+                    ("Canada/Newfoundland", "(GMT-0230) Canada/Newfoundland"),
+                    ("America/Miquelon", "(GMT-0200) America/Miquelon"),
+                    ("America/Noronha", "(GMT-0200) America/Noronha"),
+                    (
+                        "Atlantic/South_Georgia",
+                        "(GMT-0200) Atlantic/South_Georgia",
+                    ),
+                    (
+                        "America/Scoresbysund",
+                        "(GMT-0100) America/Scoresbysund",
+                    ),
+                    ("Atlantic/Azores", "(GMT-0100) Atlantic/Azores"),
+                    ("Atlantic/Cape_Verde", "(GMT-0100) Atlantic/Cape_Verde"),
+                    ("Africa/Abidjan", "(GMT+0000) Africa/Abidjan"),
+                    ("Africa/Accra", "(GMT+0000) Africa/Accra"),
+                    ("Africa/Bamako", "(GMT+0000) Africa/Bamako"),
+                    ("Africa/Banjul", "(GMT+0000) Africa/Banjul"),
+                    ("Africa/Bissau", "(GMT+0000) Africa/Bissau"),
+                    ("Africa/Conakry", "(GMT+0000) Africa/Conakry"),
+                    ("Africa/Dakar", "(GMT+0000) Africa/Dakar"),
+                    ("Africa/Freetown", "(GMT+0000) Africa/Freetown"),
+                    ("Africa/Lome", "(GMT+0000) Africa/Lome"),
+                    ("Africa/Monrovia", "(GMT+0000) Africa/Monrovia"),
+                    ("Africa/Nouakchott", "(GMT+0000) Africa/Nouakchott"),
+                    ("Africa/Ouagadougou", "(GMT+0000) Africa/Ouagadougou"),
+                    ("Africa/Sao_Tome", "(GMT+0000) Africa/Sao_Tome"),
+                    (
+                        "America/Danmarkshavn",
+                        "(GMT+0000) America/Danmarkshavn",
+                    ),
+                    ("Antarctica/Troll", "(GMT+0000) Antarctica/Troll"),
+                    ("Atlantic/Canary", "(GMT+0000) Atlantic/Canary"),
+                    ("Atlantic/Faroe", "(GMT+0000) Atlantic/Faroe"),
+                    ("Atlantic/Madeira", "(GMT+0000) Atlantic/Madeira"),
+                    ("Atlantic/Reykjavik", "(GMT+0000) Atlantic/Reykjavik"),
+                    ("Atlantic/St_Helena", "(GMT+0000) Atlantic/St_Helena"),
+                    ("Europe/Dublin", "(GMT+0000) Europe/Dublin"),
+                    ("Europe/Guernsey", "(GMT+0000) Europe/Guernsey"),
+                    ("Europe/Isle_of_Man", "(GMT+0000) Europe/Isle_of_Man"),
+                    ("Europe/Jersey", "(GMT+0000) Europe/Jersey"),
+                    ("Europe/Lisbon", "(GMT+0000) Europe/Lisbon"),
+                    ("Europe/London", "(GMT+0000) Europe/London"),
+                    ("GMT", "(GMT+0000) GMT"),
+                    ("UTC", "(GMT+0000) UTC"),
+                    ("Africa/Algiers", "(GMT+0100) Africa/Algiers"),
+                    ("Africa/Bangui", "(GMT+0100) Africa/Bangui"),
+                    ("Africa/Brazzaville", "(GMT+0100) Africa/Brazzaville"),
+                    ("Africa/Casablanca", "(GMT+0100) Africa/Casablanca"),
+                    ("Africa/Ceuta", "(GMT+0100) Africa/Ceuta"),
+                    ("Africa/Douala", "(GMT+0100) Africa/Douala"),
+                    ("Africa/El_Aaiun", "(GMT+0100) Africa/El_Aaiun"),
+                    ("Africa/Kinshasa", "(GMT+0100) Africa/Kinshasa"),
+                    ("Africa/Lagos", "(GMT+0100) Africa/Lagos"),
+                    ("Africa/Libreville", "(GMT+0100) Africa/Libreville"),
+                    ("Africa/Luanda", "(GMT+0100) Africa/Luanda"),
+                    ("Africa/Malabo", "(GMT+0100) Africa/Malabo"),
+                    ("Africa/Ndjamena", "(GMT+0100) Africa/Ndjamena"),
+                    ("Africa/Niamey", "(GMT+0100) Africa/Niamey"),
+                    ("Africa/Porto-Novo", "(GMT+0100) Africa/Porto-Novo"),
+                    ("Africa/Tunis", "(GMT+0100) Africa/Tunis"),
+                    ("Arctic/Longyearbyen", "(GMT+0100) Arctic/Longyearbyen"),
+                    ("Europe/Amsterdam", "(GMT+0100) Europe/Amsterdam"),
+                    ("Europe/Andorra", "(GMT+0100) Europe/Andorra"),
+                    ("Europe/Belgrade", "(GMT+0100) Europe/Belgrade"),
+                    ("Europe/Berlin", "(GMT+0100) Europe/Berlin"),
+                    ("Europe/Bratislava", "(GMT+0100) Europe/Bratislava"),
+                    ("Europe/Brussels", "(GMT+0100) Europe/Brussels"),
+                    ("Europe/Budapest", "(GMT+0100) Europe/Budapest"),
+                    ("Europe/Busingen", "(GMT+0100) Europe/Busingen"),
+                    ("Europe/Copenhagen", "(GMT+0100) Europe/Copenhagen"),
+                    ("Europe/Gibraltar", "(GMT+0100) Europe/Gibraltar"),
+                    ("Europe/Ljubljana", "(GMT+0100) Europe/Ljubljana"),
+                    ("Europe/Luxembourg", "(GMT+0100) Europe/Luxembourg"),
+                    ("Europe/Madrid", "(GMT+0100) Europe/Madrid"),
+                    ("Europe/Malta", "(GMT+0100) Europe/Malta"),
+                    ("Europe/Monaco", "(GMT+0100) Europe/Monaco"),
+                    ("Europe/Oslo", "(GMT+0100) Europe/Oslo"),
+                    ("Europe/Paris", "(GMT+0100) Europe/Paris"),
+                    ("Europe/Podgorica", "(GMT+0100) Europe/Podgorica"),
+                    ("Europe/Prague", "(GMT+0100) Europe/Prague"),
+                    ("Europe/Rome", "(GMT+0100) Europe/Rome"),
+                    ("Europe/San_Marino", "(GMT+0100) Europe/San_Marino"),
+                    ("Europe/Sarajevo", "(GMT+0100) Europe/Sarajevo"),
+                    ("Europe/Skopje", "(GMT+0100) Europe/Skopje"),
+                    ("Europe/Stockholm", "(GMT+0100) Europe/Stockholm"),
+                    ("Europe/Tirane", "(GMT+0100) Europe/Tirane"),
+                    ("Europe/Vaduz", "(GMT+0100) Europe/Vaduz"),
+                    ("Europe/Vatican", "(GMT+0100) Europe/Vatican"),
+                    ("Europe/Vienna", "(GMT+0100) Europe/Vienna"),
+                    ("Europe/Warsaw", "(GMT+0100) Europe/Warsaw"),
+                    ("Europe/Zagreb", "(GMT+0100) Europe/Zagreb"),
+                    ("Europe/Zurich", "(GMT+0100) Europe/Zurich"),
+                    ("Africa/Blantyre", "(GMT+0200) Africa/Blantyre"),
+                    ("Africa/Bujumbura", "(GMT+0200) Africa/Bujumbura"),
+                    ("Africa/Cairo", "(GMT+0200) Africa/Cairo"),
+                    ("Africa/Gaborone", "(GMT+0200) Africa/Gaborone"),
+                    ("Africa/Harare", "(GMT+0200) Africa/Harare"),
+                    ("Africa/Johannesburg", "(GMT+0200) Africa/Johannesburg"),
+                    ("Africa/Juba", "(GMT+0200) Africa/Juba"),
+                    ("Africa/Khartoum", "(GMT+0200) Africa/Khartoum"),
+                    ("Africa/Kigali", "(GMT+0200) Africa/Kigali"),
+                    ("Africa/Lubumbashi", "(GMT+0200) Africa/Lubumbashi"),
+                    ("Africa/Lusaka", "(GMT+0200) Africa/Lusaka"),
+                    ("Africa/Maputo", "(GMT+0200) Africa/Maputo"),
+                    ("Africa/Maseru", "(GMT+0200) Africa/Maseru"),
+                    ("Africa/Mbabane", "(GMT+0200) Africa/Mbabane"),
+                    ("Africa/Tripoli", "(GMT+0200) Africa/Tripoli"),
+                    ("Africa/Windhoek", "(GMT+0200) Africa/Windhoek"),
+                    ("Asia/Beirut", "(GMT+0200) Asia/Beirut"),
+                    ("Asia/Famagusta", "(GMT+0200) Asia/Famagusta"),
+                    ("Asia/Gaza", "(GMT+0200) Asia/Gaza"),
+                    ("Asia/Hebron", "(GMT+0200) Asia/Hebron"),
+                    ("Asia/Jerusalem", "(GMT+0200) Asia/Jerusalem"),
+                    ("Asia/Nicosia", "(GMT+0200) Asia/Nicosia"),
+                    ("Europe/Athens", "(GMT+0200) Europe/Athens"),
+                    ("Europe/Bucharest", "(GMT+0200) Europe/Bucharest"),
+                    ("Europe/Chisinau", "(GMT+0200) Europe/Chisinau"),
+                    ("Europe/Helsinki", "(GMT+0200) Europe/Helsinki"),
+                    ("Europe/Kaliningrad", "(GMT+0200) Europe/Kaliningrad"),
+                    ("Europe/Kyiv", "(GMT+0200) Europe/Kyiv"),
+                    ("Europe/Mariehamn", "(GMT+0200) Europe/Mariehamn"),
+                    ("Europe/Riga", "(GMT+0200) Europe/Riga"),
+                    ("Europe/Sofia", "(GMT+0200) Europe/Sofia"),
+                    ("Europe/Tallinn", "(GMT+0200) Europe/Tallinn"),
+                    ("Europe/Vilnius", "(GMT+0200) Europe/Vilnius"),
+                    ("Africa/Addis_Ababa", "(GMT+0300) Africa/Addis_Ababa"),
+                    ("Africa/Asmara", "(GMT+0300) Africa/Asmara"),
+                    (
+                        "Africa/Dar_es_Salaam",
+                        "(GMT+0300) Africa/Dar_es_Salaam",
+                    ),
+                    ("Africa/Djibouti", "(GMT+0300) Africa/Djibouti"),
+                    ("Africa/Kampala", "(GMT+0300) Africa/Kampala"),
+                    ("Africa/Mogadishu", "(GMT+0300) Africa/Mogadishu"),
+                    ("Africa/Nairobi", "(GMT+0300) Africa/Nairobi"),
+                    ("Antarctica/Syowa", "(GMT+0300) Antarctica/Syowa"),
+                    ("Asia/Aden", "(GMT+0300) Asia/Aden"),
+                    ("Asia/Amman", "(GMT+0300) Asia/Amman"),
+                    ("Asia/Baghdad", "(GMT+0300) Asia/Baghdad"),
+                    ("Asia/Bahrain", "(GMT+0300) Asia/Bahrain"),
+                    ("Asia/Damascus", "(GMT+0300) Asia/Damascus"),
+                    ("Asia/Kuwait", "(GMT+0300) Asia/Kuwait"),
+                    ("Asia/Qatar", "(GMT+0300) Asia/Qatar"),
+                    ("Asia/Riyadh", "(GMT+0300) Asia/Riyadh"),
+                    ("Europe/Istanbul", "(GMT+0300) Europe/Istanbul"),
+                    ("Europe/Kirov", "(GMT+0300) Europe/Kirov"),
+                    ("Europe/Minsk", "(GMT+0300) Europe/Minsk"),
+                    ("Europe/Moscow", "(GMT+0300) Europe/Moscow"),
+                    ("Europe/Simferopol", "(GMT+0300) Europe/Simferopol"),
+                    ("Europe/Volgograd", "(GMT+0300) Europe/Volgograd"),
+                    ("Indian/Antananarivo", "(GMT+0300) Indian/Antananarivo"),
+                    ("Indian/Comoro", "(GMT+0300) Indian/Comoro"),
+                    ("Indian/Mayotte", "(GMT+0300) Indian/Mayotte"),
+                    ("Asia/Tehran", "(GMT+0330) Asia/Tehran"),
+                    ("Asia/Baku", "(GMT+0400) Asia/Baku"),
+                    ("Asia/Dubai", "(GMT+0400) Asia/Dubai"),
+                    ("Asia/Muscat", "(GMT+0400) Asia/Muscat"),
+                    ("Asia/Tbilisi", "(GMT+0400) Asia/Tbilisi"),
+                    ("Asia/Yerevan", "(GMT+0400) Asia/Yerevan"),
+                    ("Europe/Astrakhan", "(GMT+0400) Europe/Astrakhan"),
+                    ("Europe/Samara", "(GMT+0400) Europe/Samara"),
+                    ("Europe/Saratov", "(GMT+0400) Europe/Saratov"),
+                    ("Europe/Ulyanovsk", "(GMT+0400) Europe/Ulyanovsk"),
+                    ("Indian/Mahe", "(GMT+0400) Indian/Mahe"),
+                    ("Indian/Mauritius", "(GMT+0400) Indian/Mauritius"),
+                    ("Indian/Reunion", "(GMT+0400) Indian/Reunion"),
+                    ("Asia/Kabul", "(GMT+0430) Asia/Kabul"),
+                    ("Antarctica/Mawson", "(GMT+0500) Antarctica/Mawson"),
+                    ("Asia/Aqtau", "(GMT+0500) Asia/Aqtau"),
+                    ("Asia/Aqtobe", "(GMT+0500) Asia/Aqtobe"),
+                    ("Asia/Ashgabat", "(GMT+0500) Asia/Ashgabat"),
+                    ("Asia/Atyrau", "(GMT+0500) Asia/Atyrau"),
+                    ("Asia/Dushanbe", "(GMT+0500) Asia/Dushanbe"),
+                    ("Asia/Karachi", "(GMT+0500) Asia/Karachi"),
+                    ("Asia/Oral", "(GMT+0500) Asia/Oral"),
+                    ("Asia/Qyzylorda", "(GMT+0500) Asia/Qyzylorda"),
+                    ("Asia/Samarkand", "(GMT+0500) Asia/Samarkand"),
+                    ("Asia/Tashkent", "(GMT+0500) Asia/Tashkent"),
+                    ("Asia/Yekaterinburg", "(GMT+0500) Asia/Yekaterinburg"),
+                    ("Indian/Kerguelen", "(GMT+0500) Indian/Kerguelen"),
+                    ("Indian/Maldives", "(GMT+0500) Indian/Maldives"),
+                    ("Asia/Colombo", "(GMT+0530) Asia/Colombo"),
+                    ("Asia/Kolkata", "(GMT+0530) Asia/Kolkata"),
+                    ("Asia/Kathmandu", "(GMT+0545) Asia/Kathmandu"),
+                    ("Antarctica/Vostok", "(GMT+0600) Antarctica/Vostok"),
+                    ("Asia/Almaty", "(GMT+0600) Asia/Almaty"),
+                    ("Asia/Bishkek", "(GMT+0600) Asia/Bishkek"),
+                    ("Asia/Dhaka", "(GMT+0600) Asia/Dhaka"),
+                    ("Asia/Omsk", "(GMT+0600) Asia/Omsk"),
+                    ("Asia/Qostanay", "(GMT+0600) Asia/Qostanay"),
+                    ("Asia/Thimphu", "(GMT+0600) Asia/Thimphu"),
+                    ("Asia/Urumqi", "(GMT+0600) Asia/Urumqi"),
+                    ("Indian/Chagos", "(GMT+0600) Indian/Chagos"),
+                    ("Asia/Yangon", "(GMT+0630) Asia/Yangon"),
+                    ("Indian/Cocos", "(GMT+0630) Indian/Cocos"),
+                    ("Antarctica/Davis", "(GMT+0700) Antarctica/Davis"),
+                    ("Asia/Bangkok", "(GMT+0700) Asia/Bangkok"),
+                    ("Asia/Barnaul", "(GMT+0700) Asia/Barnaul"),
+                    ("Asia/Ho_Chi_Minh", "(GMT+0700) Asia/Ho_Chi_Minh"),
+                    ("Asia/Hovd", "(GMT+0700) Asia/Hovd"),
+                    ("Asia/Jakarta", "(GMT+0700) Asia/Jakarta"),
+                    ("Asia/Krasnoyarsk", "(GMT+0700) Asia/Krasnoyarsk"),
+                    ("Asia/Novokuznetsk", "(GMT+0700) Asia/Novokuznetsk"),
+                    ("Asia/Novosibirsk", "(GMT+0700) Asia/Novosibirsk"),
+                    ("Asia/Phnom_Penh", "(GMT+0700) Asia/Phnom_Penh"),
+                    ("Asia/Pontianak", "(GMT+0700) Asia/Pontianak"),
+                    ("Asia/Tomsk", "(GMT+0700) Asia/Tomsk"),
+                    ("Asia/Vientiane", "(GMT+0700) Asia/Vientiane"),
+                    ("Indian/Christmas", "(GMT+0700) Indian/Christmas"),
+                    ("Asia/Brunei", "(GMT+0800) Asia/Brunei"),
+                    ("Asia/Choibalsan", "(GMT+0800) Asia/Choibalsan"),
+                    ("Asia/Hong_Kong", "(GMT+0800) Asia/Hong_Kong"),
+                    ("Asia/Irkutsk", "(GMT+0800) Asia/Irkutsk"),
+                    ("Asia/Kuala_Lumpur", "(GMT+0800) Asia/Kuala_Lumpur"),
+                    ("Asia/Kuching", "(GMT+0800) Asia/Kuching"),
+                    ("Asia/Macau", "(GMT+0800) Asia/Macau"),
+                    ("Asia/Makassar", "(GMT+0800) Asia/Makassar"),
+                    ("Asia/Manila", "(GMT+0800) Asia/Manila"),
+                    ("Asia/Shanghai", "(GMT+0800) Asia/Shanghai"),
+                    ("Asia/Singapore", "(GMT+0800) Asia/Singapore"),
+                    ("Asia/Taipei", "(GMT+0800) Asia/Taipei"),
+                    ("Asia/Ulaanbaatar", "(GMT+0800) Asia/Ulaanbaatar"),
+                    ("Australia/Perth", "(GMT+0800) Australia/Perth"),
+                    ("Australia/Eucla", "(GMT+0845) Australia/Eucla"),
+                    ("Asia/Chita", "(GMT+0900) Asia/Chita"),
+                    ("Asia/Dili", "(GMT+0900) Asia/Dili"),
+                    ("Asia/Jayapura", "(GMT+0900) Asia/Jayapura"),
+                    ("Asia/Khandyga", "(GMT+0900) Asia/Khandyga"),
+                    ("Asia/Pyongyang", "(GMT+0900) Asia/Pyongyang"),
+                    ("Asia/Seoul", "(GMT+0900) Asia/Seoul"),
+                    ("Asia/Tokyo", "(GMT+0900) Asia/Tokyo"),
+                    ("Asia/Yakutsk", "(GMT+0900) Asia/Yakutsk"),
+                    ("Pacific/Palau", "(GMT+0900) Pacific/Palau"),
+                    ("Australia/Darwin", "(GMT+0930) Australia/Darwin"),
+                    (
+                        "Antarctica/DumontDUrville",
+                        "(GMT+1000) Antarctica/DumontDUrville",
+                    ),
+                    ("Asia/Ust-Nera", "(GMT+1000) Asia/Ust-Nera"),
+                    ("Asia/Vladivostok", "(GMT+1000) Asia/Vladivostok"),
+                    ("Australia/Brisbane", "(GMT+1000) Australia/Brisbane"),
+                    ("Australia/Lindeman", "(GMT+1000) Australia/Lindeman"),
+                    ("Pacific/Chuuk", "(GMT+1000) Pacific/Chuuk"),
+                    ("Pacific/Guam", "(GMT+1000) Pacific/Guam"),
+                    (
+                        "Pacific/Port_Moresby",
+                        "(GMT+1000) Pacific/Port_Moresby",
+                    ),
+                    ("Pacific/Saipan", "(GMT+1000) Pacific/Saipan"),
+                    ("Australia/Adelaide", "(GMT+1030) Australia/Adelaide"),
+                    (
+                        "Australia/Broken_Hill",
+                        "(GMT+1030) Australia/Broken_Hill",
+                    ),
+                    ("Antarctica/Casey", "(GMT+1100) Antarctica/Casey"),
+                    (
+                        "Antarctica/Macquarie",
+                        "(GMT+1100) Antarctica/Macquarie",
+                    ),
+                    ("Asia/Magadan", "(GMT+1100) Asia/Magadan"),
+                    ("Asia/Sakhalin", "(GMT+1100) Asia/Sakhalin"),
+                    ("Asia/Srednekolymsk", "(GMT+1100) Asia/Srednekolymsk"),
+                    ("Australia/Hobart", "(GMT+1100) Australia/Hobart"),
+                    ("Australia/Lord_Howe", "(GMT+1100) Australia/Lord_Howe"),
+                    ("Australia/Melbourne", "(GMT+1100) Australia/Melbourne"),
+                    ("Australia/Sydney", "(GMT+1100) Australia/Sydney"),
+                    (
+                        "Pacific/Bougainville",
+                        "(GMT+1100) Pacific/Bougainville",
+                    ),
+                    ("Pacific/Efate", "(GMT+1100) Pacific/Efate"),
+                    ("Pacific/Guadalcanal", "(GMT+1100) Pacific/Guadalcanal"),
+                    ("Pacific/Kosrae", "(GMT+1100) Pacific/Kosrae"),
+                    ("Pacific/Noumea", "(GMT+1100) Pacific/Noumea"),
+                    ("Pacific/Pohnpei", "(GMT+1100) Pacific/Pohnpei"),
+                    ("Asia/Anadyr", "(GMT+1200) Asia/Anadyr"),
+                    ("Asia/Kamchatka", "(GMT+1200) Asia/Kamchatka"),
+                    ("Pacific/Fiji", "(GMT+1200) Pacific/Fiji"),
+                    ("Pacific/Funafuti", "(GMT+1200) Pacific/Funafuti"),
+                    ("Pacific/Kwajalein", "(GMT+1200) Pacific/Kwajalein"),
+                    ("Pacific/Majuro", "(GMT+1200) Pacific/Majuro"),
+                    ("Pacific/Nauru", "(GMT+1200) Pacific/Nauru"),
+                    ("Pacific/Norfolk", "(GMT+1200) Pacific/Norfolk"),
+                    ("Pacific/Tarawa", "(GMT+1200) Pacific/Tarawa"),
+                    ("Pacific/Wake", "(GMT+1200) Pacific/Wake"),
+                    ("Pacific/Wallis", "(GMT+1200) Pacific/Wallis"),
+                    ("Antarctica/McMurdo", "(GMT+1300) Antarctica/McMurdo"),
+                    ("Pacific/Apia", "(GMT+1300) Pacific/Apia"),
+                    ("Pacific/Auckland", "(GMT+1300) Pacific/Auckland"),
+                    ("Pacific/Fakaofo", "(GMT+1300) Pacific/Fakaofo"),
+                    ("Pacific/Kanton", "(GMT+1300) Pacific/Kanton"),
+                    ("Pacific/Tongatapu", "(GMT+1300) Pacific/Tongatapu"),
+                    ("Pacific/Chatham", "(GMT+1345) Pacific/Chatham"),
+                    ("Pacific/Kiritimati", "(GMT+1400) Pacific/Kiritimati"),
+                ],
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

vrobbler/apps/profiles/migrations/0007_alter_userprofile_timezone.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.1.5 on 2023-03-22 22:26
+
+from django.db import migrations, models
+from profiles.constants import PRETTY_TIMEZONE_CHOICES
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0006_alter_userprofile_timezone"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="userprofile",
+            name="timezone",
+            field=models.CharField(
+                blank=True,
+                choices=PRETTY_TIMEZONE_CHOICES,
+                max_length=255,
+                null=True,
+            ),
+        ),
+    ]

vrobbler/apps/profiles/models.py

@@ -1,11 +1,12 @@
-import pytz
+from datetime import timedelta
 
+import pytz
+from django.utils import timezone
 from django.contrib.auth import get_user_model
 from django.db import models
 from django_extensions.db.models import TimeStampedModel
-from profiles.constants import PRETTY_TIMEZONE_CHOICES
-
 from encrypted_field import EncryptedField
+from profiles.constants import PRETTY_TIMEZONE_CHOICES
 
 User = get_user_model()
 BNULL = {"blank": True, "null": True}
@@ -16,7 +17,9 @@ class UserProfile(TimeStampedModel):
         User, on_delete=models.CASCADE, related_name="profile"
     )
     timezone = models.CharField(
-        max_length=255, choices=PRETTY_TIMEZONE_CHOICES
+        max_length=255,
+        choices=PRETTY_TIMEZONE_CHOICES,
+        **BNULL,
     )
     lastfm_username = models.CharField(max_length=255, **BNULL)
     lastfm_password = EncryptedField(**BNULL)

vrobbler/apps/profiles/utils.py

@@ -1,23 +1,18 @@
-import datetime
-
 import pytz
 from django.conf import settings
 from django.utils import timezone
+import calendar
 
+from datetime import datetime, timedelta
 
 # need to translate to a non-naive timezone, even if timezone == settings.TIME_ZONE, so we can compare two dates
 def to_user_timezone(date, profile):
     timezone = profile.timezone if profile.timezone else settings.TIME_ZONE
-    return date.replace(tzinfo=pytz.timezone(settings.TIME_ZONE)).astimezone(
+    return date.astimezone(pytz.timezone(timezone))
-        pytz.timezone(timezone)
-    )
 
 
-def to_system_timezone(date, profile):
+def to_system_timezone(date):
-    timezone = profile.timezone if profile.timezone else settings.TIME_ZONE
+    return date.astimezone(pytz.timezone(settings.TIME_ZONE))
-    return date.replace(tzinfo=pytz.timezone(timezone)).astimezone(
-        pytz.timezone(settings.TIME_ZONE)
-    )
 
 
 def now_user_timezone(profile):
@@ -25,9 +20,39 @@ def now_user_timezone(profile):
     return timezone.localtime(timezone.now())
 
 
-def now_system_timezone():
+def start_of_day(dt, profile) -> datetime:
-    return (
+    """Get the start of the day in the profile's timezone"""
-        datetime.datetime.now()
+    timezone = profile.timezone if profile.timezone else settings.TIME_ZONE
-        .replace(tzinfo=pytz.timezone(settings.TIME_ZONE))
+    tzinfo = pytz.timezone(timezone)
-        .astimezone(pytz.timezone(settings.TIME_ZONE))
+    return datetime.combine(dt, datetime.min.time(), tzinfo)
-    )
+
+
+def end_of_day(dt, profile) -> datetime:
+    """Get the start of the day in the profile's timezone"""
+    timezone = profile.timezone if profile.timezone else settings.TIME_ZONE
+    tzinfo = pytz.timezone(timezone)
+    return datetime.combine(dt, datetime.max.time(), tzinfo)
+
+
+def start_of_week(dt, profile) -> datetime:
+    # TODO allow profile to set start of week
+    return start_of_day(dt, profile) - timedelta(dt.weekday())
+
+
+def end_of_week(dt, profile) -> datetime:
+    # TODO allow profile to set start of week
+    return start_of_week(dt, profile) + timedelta(days=6)
+
+
+def start_of_month(dt, profile) -> datetime:
+    return start_of_day(dt, profile).replace(day=1)
+
+
+def end_of_month(dt, profile) -> datetime:
+    next_month = end_of_day(dt, profile).replace(day=28) + timedelta(days=4)
+    # subtracting the number of the current day brings us back one month
+    return next_month - timedelta(days=next_month.day)
+
+
+def start_of_year(dt, profile) -> datetime:
+    return start_of_day(dt, profile).replace(month=1, day=1)

vrobbler/apps/scrobbles/admin.py

@@ -2,27 +2,29 @@ from django.contrib import admin
 from scrobbles.models import (
     AudioScrobblerTSVImport,
     ChartRecord,
+    KoReaderImport,
     LastFmImport,
     Scrobble,
 )
+from scrobbles.mixins import Genre
 
 
 class ScrobbleInline(admin.TabularInline):
     model = Scrobble
     extra = 0
-    raw_id_fields = ('video', 'podcast_episode', 'track')
+    raw_id_fields = (
-    exclude = ('source_id', 'scrobble_log')
+        "video",
+        "podcast_episode",
+        "track",
+        "video_game",
+        "book",
+        "sport_event",
+        "user",
+    )
+    exclude = ("source_id", "scrobble_log")
 
 
-@admin.register(AudioScrobblerTSVImport)
+class ImportBaseAdmin(admin.ModelAdmin):
-class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
-    date_hierarchy = "created"
-    list_display = ("uuid", "created", "process_count", "tsv_file")
-    ordering = ("-created",)
-
-
-@admin.register(LastFmImport)
-class LastFmImportAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     list_display = (
         "uuid",
@@ -33,12 +35,36 @@ class LastFmImportAdmin(admin.ModelAdmin):
     ordering = ("-created",)
 
 
+@admin.register(AudioScrobblerTSVImport)
+class AudioScrobblerTSVImportAdmin(ImportBaseAdmin):
+    """"""
+
+
+@admin.register(LastFmImport)
+class LastFmImportAdmin(ImportBaseAdmin):
+    """"""
+
+
+@admin.register(KoReaderImport)
+class KoReaderImportAdmin(ImportBaseAdmin):
+    """"""
+
+
+@admin.register(Genre)
+class GenreAdmin(admin.ModelAdmin):
+    list_display = (
+        "name",
+        "source",
+    )
+
+
 @admin.register(ChartRecord)
 class ChartRecordAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     list_display = (
         "user",
         "rank",
+        "count",
         "year",
         "week",
         "month",
@@ -48,14 +74,7 @@ class ChartRecordAdmin(admin.ModelAdmin):
     ordering = ("-created",)
 
     def media_name(self, obj):
-        if obj.video:
+        return obj.media_obj
-            return obj.video
-        if obj.track:
-            return obj.track
-        if obj.podcast_episode:
-            return obj.podcast_episode
-        if obj.sport_event:
-            return obj.sport_event
 
 
 @admin.register(Scrobble)
@@ -71,29 +90,27 @@ class ScrobbleAdmin(admin.ModelAdmin):
         "is_paused",
         "played_to_completion",
     )
-    raw_id_fields = ('video', 'podcast_episode', 'track', 'sport_event')
+    raw_id_fields = (
-    list_filter = ("is_paused", "in_progress", "source", "track__artist")
+        "video",
+        "podcast_episode",
+        "track",
+        "sport_event",
+        "book",
+        "video_game",
+    )
+    list_filter = (
+        "is_paused",
+        "in_progress",
+        "long_play_complete",
+        "source",
+    )
     ordering = ("-timestamp",)
 
     def media_name(self, obj):
-        if obj.video:
+        return obj.media_obj
-            return obj.video
-        if obj.track:
-            return obj.track
-        if obj.podcast_episode:
-            return obj.podcast_episode
-        if obj.sport_event:
-            return obj.sport_event
 
     def media_type(self, obj):
-        if obj.video:
+        return obj.media_obj.__class__.__name__
-            return "Video"
-        if obj.track:
-            return "Track"
-        if obj.podcast_episode:
-            return "Podcast"
-        if obj.sport_event:
-            return "Sport Event"
 
     def playback_percent(self, obj):
         return obj.percent_played

vrobbler/apps/scrobbles/api/serializers.py

@@ -0,0 +1,33 @@
+from rest_framework import serializers
+from scrobbles.models import (
+    AudioScrobblerTSVImport,
+    KoReaderImport,
+    LastFmImport,
+    Scrobble,
+)
+
+
+class ScrobbleSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Scrobble
+        fields = "__all__"
+
+
+class KoReaderImportSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = KoReaderImport
+        fields = "__all__"
+
+
+class AudioScrobblerTSVImportSerializer(
+    serializers.HyperlinkedModelSerializer
+):
+    class Meta:
+        model = AudioScrobblerTSVImport
+        fields = "__all__"
+
+
+class LastFmImportSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = LastFmImport
+        fields = "__all__"

vrobbler/apps/scrobbles/api/views.py

@@ -0,0 +1,49 @@
+from rest_framework import permissions, viewsets
+from scrobbles.api.serializers import (
+    AudioScrobblerTSVImportSerializer,
+    KoReaderImportSerializer,
+    LastFmImportSerializer,
+    ScrobbleSerializer,
+)
+from scrobbles.models import (
+    AudioScrobblerTSVImport,
+    KoReaderImport,
+    Scrobble,
+    LastFmImport,
+)
+
+
+class ScrobbleViewSet(viewsets.ModelViewSet):
+    queryset = Scrobble.objects.all().order_by("-timestamp")
+    serializer_class = ScrobbleSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self):
+        return super().get_queryset().filter(user=self.request.user)
+
+
+class KoReaderImportViewSet(viewsets.ModelViewSet):
+    queryset = KoReaderImport.objects.all().order_by("-created")
+    serializer_class = KoReaderImportSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self):
+        return super().get_queryset().filter(user=self.request.user)
+
+
+class AudioScrobblerTSVImportViewSet(viewsets.ModelViewSet):
+    queryset = AudioScrobblerTSVImport.objects.all().order_by("-created")
+    serializer_class = AudioScrobblerTSVImportSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self):
+        return super().get_queryset().filter(user=self.request.user)
+
+
+class LastFmImportViewSet(viewsets.ModelViewSet):
+    queryset = LastFmImport.objects.all().order_by("-created")
+    serializer_class = LastFmImportSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self):
+        return super().get_queryset().filter(user=self.request.user)

vrobbler/apps/scrobbles/apps.py

@@ -2,4 +2,4 @@ from django.apps import AppConfig
 
 
 class ScrobblesConfig(AppConfig):
-    name = 'scrobbles'
+    name = "scrobbles"

vrobbler/apps/scrobbles/constants.py

@@ -1,2 +1,20 @@
+from enum import Enum
+
 JELLYFIN_VIDEO_ITEM_TYPES = ["Episode", "Movie"]
 JELLYFIN_AUDIO_ITEM_TYPES = ["Audio"]
+
+LONG_PLAY_MEDIA = {
+    "videogames": "VideoGame",
+    "books": "Book",
+}
+
+
+class AsTsvColumn(Enum):
+    ARTIST_NAME = 0
+    ALBUM_NAME = 1
+    TRACK_NAME = 2
+    TRACK_NUMBER = 3
+    RUN_TIME_SECONDS = 4
+    COMPLETE = 5
+    TIMESTAMP = 6
+    MB_ID = 7

vrobbler/apps/scrobbles/context_processors.py

@@ -0,0 +1,17 @@
+import pytz
+from django.utils import timezone
+from scrobbles.models import Scrobble
+
+
+def now_playing(request):
+    user = request.user
+    now = timezone.now()
+    if not user.is_authenticated:
+        return {}
+    return {
+        "now_playing_list": Scrobble.objects.filter(
+            in_progress=True,
+            is_paused=False,
+            user=user,
+        )
+    }

vrobbler/apps/scrobbles/export.py

@@ -17,19 +17,19 @@ def export_scrobbles(start_date=None, end_date=None, format="AS"):
         start_query, end_query, track__isnull=False
     )
     headers = []
-    extension = 'tsv'
+    extension = "tsv"
-    delimiter = '\t'
+    delimiter = "\t"
 
     if format == "as":
         headers = [
-            ['#AUDIOSCROBBLER/1.1'],
+            ["#AUDIOSCROBBLER/1.1"],
-            ['#TZ/UTC'],
+            ["#TZ/UTC"],
-            ['#CLIENT/Vrobbler 1.0.0'],
+            ["#CLIENT/Vrobbler 1.0.0"],
         ]
 
     if format == "csv":
-        delimiter = ','
+        delimiter = ","
-        extension = 'csv'
+        extension = "csv"
         headers = [
             [
                 "artists",
@@ -43,7 +43,7 @@ def export_scrobbles(start_date=None, end_date=None, format="AS"):
             ]
         ]
 
-    with tempfile.NamedTemporaryFile(mode='w', delete=False) as outfile:
+    with tempfile.NamedTemporaryFile(mode="w", delete=False) as outfile:
         writer = csv.writer(outfile, delimiter=delimiter)
         for row in headers:
             writer.writerow(row)
@@ -52,7 +52,7 @@ def export_scrobbles(start_date=None, end_date=None, format="AS"):
             track = scrobble.track
             track_number = 0  # TODO Add track number
             track_rating = "S"  # TODO implement ratings?
-            track_artist = track.artist or track.album.primary_artist
+            track_artist = track.artist or track.album.album_artist
             row = [
                 track_artist,
                 track.album.name,
@@ -60,7 +60,7 @@ def export_scrobbles(start_date=None, end_date=None, format="AS"):
                 track_number,
                 track.run_time,
                 track_rating,
-                scrobble.timestamp.strftime('%s'),
+                scrobble.timestamp.strftime("%s"),
                 track.musicbrainz_id,
             ]
             writer.writerow(row)

vrobbler/apps/scrobbles/forms.py

@@ -5,9 +5,9 @@ class ExportScrobbleForm(forms.Form):
     """Provide options for downloading scrobbles"""
 
     EXPORT_TYPES = (
-        ('as', 'Audioscrobbler'),
+        ("as", "Audioscrobbler"),
-        ('csv', 'CSV'),
+        ("csv", "CSV"),
-        ('html', 'HTML'),
+        ("html", "HTML"),
     )
     export_type = forms.ChoiceField(choices=EXPORT_TYPES)
 
@@ -17,9 +17,9 @@ class ScrobbleForm(forms.Form):
         label="",
         widget=forms.TextInput(
             attrs={
-                'class': "form-control form-control-dark w-100",
+                "class": "form-control form-control-dark w-100",
-                'placeholder': "Scrobble something (IMDB ID, String, TVDB ID ...)",
+                "placeholder": "Scrobble something (ttIMDB, -v Video Game title, -b Book title, -s TheSportsDB ID)",
-                'aria-label': "Scrobble something",
+                "aria-label": "Scrobble something",
             }
         ),
     )

vrobbler/apps/scrobbles/imdb.py

@@ -1,61 +0,0 @@
-import logging
-from django.utils import timezone
-
-from imdb import Cinemagoer
-
-imdb_client = Cinemagoer()
-
-logger = logging.getLogger(__name__)
-
-
-def lookup_video_from_imdb(imdb_id: str) -> dict:
-
-    if 'tt' not in imdb_id:
-        logger.warning(f"IMDB ID should begin with 'tt' {imdb_id}")
-        return
-
-    lookup_id = imdb_id.strip('tt')
-    media = imdb_client.get_movie(lookup_id)
-
-    run_time_seconds = 60 * 60
-    runtimes = media.get("runtimes")
-    if runtimes:
-        run_time_seconds = int(runtimes[0]) * 60
-
-    # Ticks otherwise known as miliseconds
-    run_time_ticks = run_time_seconds * 1000 * 1000
-
-    item_type = "Movie"
-    if media.get('series title'):
-        item_type = "Episode"
-
-    try:
-        plot = media.get('plot')[0]
-    except TypeError:
-        plot = ""
-    except IndexError:
-        plot = ""
-
-    logger.debug(f"Received data from IMDB: {media.__dict__}")
-    # Build a rough approximation of a Jellyfin data response
-    data_dict = {
-        "ItemType": item_type,
-        "Name": media.get('title'),
-        "Overview": plot,
-        "Tagline": media.get('tagline'),
-        "Year": media.get('year'),
-        "Provider_imdb": imdb_id,
-        "RunTime": run_time_seconds,
-        "RunTimeTicks": run_time_ticks,
-        "SeriesName": media.get('series title'),
-        "EpisodeNumber": media.get('episode'),
-        "SeasonNumber": media.get('season'),
-        "PlaybackPositionTicks": 1,
-        "PlaybackPosition": 1,
-        "UtcTimestamp": timezone.now().strftime('%Y-%m-%d %H:%M:%S.%f%z'),
-        "IsPaused": False,
-        "PlayedToCompletion": False,
-    }
-    logger.debug(f"Parsed data from IMDB data: {data_dict}")
-
-    return data_dict