[release] Bump to version 51.3

- Improve speed of index and chart pages

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [0/14] :vrobbler:project:personal:
+* Backlog [0/15] :vrobbler:project:personal:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles:
 :PROPERTIES:
 :ID:       702462cf-d54b-48c6-8a7c-78b8de751deb
@@ -518,6 +518,33 @@ favorited media objects.
 :PROPERTIES:
 :ID:       79f867c3-1288-4143-b6bf-2a452983ee9f
 :END:
+** TODO [#C] Implement loguru into project :feature:loguru:logging:
+:PROPERTIES:
+:ID:       efcd0c0a-db81-4518-9c23-5505d59e8ef5
+:END:
+
+*** Description
+
+Would be great to formalize how we log so we can search for errors and such more
+easily. And our exposure to PII is really low at this point in the project,
+so we can probably use backtrace=True and diagnose=True to help us root cause
+bugs faster.
+
+* Version 51.3 [1/1]
+** DONE [#A] Improve speed of index and chart pages :bug:scrobbles:perf:
+:PROPERTIES:
+:ID:       031a23f8-7c02-4926-9884-6654ceca16c2
+:END:
+
+*** Description
+
+Over the last few releases, the home page and charts pages have gotten really
+slow.
+
+We should look into what's causing the slowness and maybe do more agressive
+query optimization or caching.
+
+
 * Version 51.2 [2/2]
 ** DONE [#A] Fix bug where last page of book gets separate scrobble :bug:books:importers:koreader:
 :PROPERTIES:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vrobbler"
-version = "51.2"
+version = "51.3"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 

vrobbler/apps/charts/migrations/0002_chartrecord_charts_char_user_id_1adcde_idx_and_more.py

@@ -0,0 +1,146 @@
+# Generated by Django 4.2.29 on 2026-06-12 16:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("charts", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "album", "rank"],
+                name="charts_char_user_id_1adcde_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "track", "rank"],
+                name="charts_char_user_id_d18aab_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "video", "rank"],
+                name="charts_char_user_id_de9f0a_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "board_game", "rank"],
+                name="charts_char_user_id_d5d58f_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "book", "rank"],
+                name="charts_char_user_id_e877cf_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "food", "rank"],
+                name="charts_char_user_id_a0ad71_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "podcast", "rank"],
+                name="charts_char_user_id_846b80_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "trail", "rank"],
+                name="charts_char_user_id_54feba_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "album", "rank"],
+                name="charts_char_user_id_a3dc49_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "track", "rank"],
+                name="charts_char_user_id_4b01ab_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "video", "rank"],
+                name="charts_char_user_id_2ac9d2_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "board_game", "rank"],
+                name="charts_char_user_id_ba968a_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "book", "rank"],
+                name="charts_char_user_id_e66751_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "food", "rank"],
+                name="charts_char_user_id_d23f06_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "podcast", "rank"],
+                name="charts_char_user_id_be8122_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "week", "trail", "rank"],
+                name="charts_char_user_id_b94ea9_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "day", "artist", "rank"],
+                name="charts_char_user_id_406e0e_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "day", "album", "rank"],
+                name="charts_char_user_id_322b0d_idx",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="chartrecord",
+            index=models.Index(
+                fields=["user", "year", "month", "day", "tv_series", "rank"],
+                name="charts_char_user_id_aa44b7_idx",
+            ),
+        ),
+    ]

vrobbler/apps/charts/models.py

@@ -60,10 +60,29 @@ class ChartRecord(TimeStampedModel):
             models.Index(fields=["user", "year", "geo_location", "rank"]),
             models.Index(fields=["user", "year", "food", "rank"]),
             models.Index(fields=["user", "year", "book", "rank"]),
-            models.Index(fields=["user", "year", "week", "artist", "rank"]),
-            models.Index(fields=["user", "year", "week", "tv_series", "rank"]),
             models.Index(fields=["user", "year", "month", "artist", "rank"]),
+            models.Index(fields=["user", "year", "month", "album", "rank"]),
+            models.Index(fields=["user", "year", "month", "track", "rank"]),
             models.Index(fields=["user", "year", "month", "tv_series", "rank"]),
+            models.Index(fields=["user", "year", "month", "video", "rank"]),
+            models.Index(fields=["user", "year", "month", "board_game", "rank"]),
+            models.Index(fields=["user", "year", "month", "book", "rank"]),
+            models.Index(fields=["user", "year", "month", "food", "rank"]),
+            models.Index(fields=["user", "year", "month", "podcast", "rank"]),
+            models.Index(fields=["user", "year", "month", "trail", "rank"]),
+            models.Index(fields=["user", "year", "week", "artist", "rank"]),
+            models.Index(fields=["user", "year", "week", "album", "rank"]),
+            models.Index(fields=["user", "year", "week", "track", "rank"]),
+            models.Index(fields=["user", "year", "week", "tv_series", "rank"]),
+            models.Index(fields=["user", "year", "week", "video", "rank"]),
+            models.Index(fields=["user", "year", "week", "board_game", "rank"]),
+            models.Index(fields=["user", "year", "week", "book", "rank"]),
+            models.Index(fields=["user", "year", "week", "food", "rank"]),
+            models.Index(fields=["user", "year", "week", "podcast", "rank"]),
+            models.Index(fields=["user", "year", "week", "trail", "rank"]),
+            models.Index(fields=["user", "year", "month", "day", "artist", "rank"]),
+            models.Index(fields=["user", "year", "month", "day", "album", "rank"]),
+            models.Index(fields=["user", "year", "month", "day", "tv_series", "rank"]),
         ]
 
     @property

vrobbler/apps/charts/views.py

@@ -438,12 +438,14 @@ class ChartRecordView(TemplateView):
         return context
 
     def get_available_years(self, user):
-        return list(
-            ChartRecord.objects.filter(user=user)
-            .values_list("year", flat=True)
-            .distinct()
-            .order_by("-year")
-        )
+        if not hasattr(self, "_available_years"):
+            self._available_years = list(
+                ChartRecord.objects.filter(user=user)
+                .values_list("year", flat=True)
+                .distinct()
+                .order_by("-year")
+            )
+        return self._available_years
 
     def get_period_type(self):
         date_param = self.request.GET.get("date")

vrobbler/apps/scrobbles/migrations/0097_scrobble_scrobbles_s_user_id_d367a7_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.29 on 2026-06-12 16:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0096_convert_book_page_data_to_dict"),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="scrobble",
+            index=models.Index(
+                fields=["user", "-timestamp"], name="scrobbles_s_user_id_d367a7_idx"
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -612,6 +612,30 @@ class EBirdCSVImport(BaseFileImportMixin):
             self.mark_finished()
 
 
+TYPE_FK_PREFETCHES: dict[str, tuple[str, ...]] = {
+    "Video": ("video",),
+    "Track": ("track", "track__artist_fk"),
+    "PodcastEpisode": ("podcast_episode", "podcast_episode__podcast"),
+    "SportEvent": ("sport_event",),
+    "Book": ("book",),
+    "Paper": ("paper",),
+    "VideoGame": ("video_game",),
+    "BoardGame": ("board_game",),
+    "GeoLocation": ("geo_location",),
+    "Trail": ("trail",),
+    "Beer": ("beer",),
+    "Puzzle": ("puzzle",),
+    "Food": ("food",),
+    "Task": ("task",),
+    "WebPage": ("web_page",),
+    "LifeEvent": ("life_event",),
+    "Mood": ("mood",),
+    "BrickSet": ("brick_set",),
+    "Channel": ("channel",),
+    "BirdingLocation": ("birding_location",),
+}
+
+
 class ScrobbleQuerySet(models.QuerySet):
     def with_related(self):
         return self.select_related("user").prefetch_related(
@@ -638,6 +662,13 @@ class ScrobbleQuerySet(models.QuerySet):
             "birding_location",
         )
 
+    def with_related_for_types(self, media_types: list[str]):
+        prefetches = []
+        for t in media_types:
+            if t in TYPE_FK_PREFETCHES:
+                prefetches.extend(TYPE_FK_PREFETCHES[t])
+        return self.select_related("user").prefetch_related(*prefetches)
+
 
 class ShareViewLog(TimeStampedModel):
     scrobble = models.ForeignKey(
@@ -778,6 +809,7 @@ class Scrobble(TimeStampedModel):
                     "is_paused",
                 ]
             ),
+            models.Index(fields=["user", "-timestamp"]),
         ]
 
     @classmethod
@@ -810,11 +842,14 @@ class Scrobble(TimeStampedModel):
     @classmethod
     def as_dict_by_type(cls, scrobble_qs: models.QuerySet) -> dict:
         scrobbles_by_type = defaultdict(list)
-        scrobbles = (
-            scrobble_qs.with_related()
-            if hasattr(scrobble_qs, "with_related")
-            else scrobble_qs
-        )
+
+        if hasattr(scrobble_qs, "with_related"):
+            media_types_present = list(
+                scrobble_qs.values_list("media_type", flat=True).distinct()
+            )
+            scrobbles = scrobble_qs.with_related_for_types(media_types_present)
+        else:
+            scrobbles = scrobble_qs
 
         for scrobble in scrobbles:
             scrobbles_by_type[scrobble.media_type].append(scrobble)
@@ -829,7 +864,7 @@ class Scrobble(TimeStampedModel):
 
         # Remove any locations without titles
         if "GeoLocation" in scrobbles_by_type.keys():
-            for loc_scrobble in scrobbles_by_type["GeoLocation"]:
+            for loc_scrobble in list(scrobbles_by_type["GeoLocation"]):
                 if not loc_scrobble.media_obj.title:
                     scrobbles_by_type["GeoLocation"].remove(loc_scrobble)
                     scrobbles_by_type["GeoLocation_count"] -= 1