From c7f5d7d38432d593ad7e1a8fa842d715ab08ee26 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Fri, 12 Jun 2026 13:34:41 -0400 Subject: [PATCH] [charts] Add index to speed things up --- PROJECT.org | 27 +++- ...charts_char_user_id_1adcde_idx_and_more.py | 146 ++++++++++++++++++ vrobbler/apps/charts/models.py | 23 ++- vrobbler/apps/charts/views.py | 14 +- ...scrobble_scrobbles_s_user_id_d367a7_idx.py | 19 +++ vrobbler/apps/scrobbles/models.py | 47 +++++- 6 files changed, 261 insertions(+), 15 deletions(-) create mode 100644 vrobbler/apps/charts/migrations/0002_chartrecord_charts_char_user_id_1adcde_idx_and_more.py create mode 100644 vrobbler/apps/scrobbles/migrations/0097_scrobble_scrobbles_s_user_id_d367a7_idx.py diff --git a/PROJECT.org b/PROJECT.org index defecbb..93c4c1d 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/14] :vrobbler:project:personal: +* Backlog [1/16] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -518,6 +518,31 @@ favorited media objects. :PROPERTIES: :ID: 79f867c3-1288-4143-b6bf-2a452983ee9f :END: +** TODO [#C] Implement loguru into project :feature:loguru:logging: +:PROPERTIES: +:ID: efcd0c0a-db81-4518-9c23-5505d59e8ef5 +:END: + +*** Description + +Would be great to formalize how we log so we can search for errors and such more +easily. And our exposure to PII is really low at this point in the project, +so we can probably use backtrace=True and diagnose=True to help us root cause +bugs faster. + +** DONE [#A] Improve speed of index and chart pages :bug:scrobbles:perf: +:PROPERTIES: +:ID: 031a23f8-7c02-4926-9884-6654ceca16c2 +:END: + +*** Description + +Over the last few releases, the home page and charts pages have gotten really +slow. + +We should look into what's causing the slowness and maybe do more agressive +query optimization or caching. + * Version 51.2 [2/2] ** DONE [#A] Fix bug where last page of book gets separate scrobble :bug:books:importers:koreader: :PROPERTIES: diff --git a/vrobbler/apps/charts/migrations/0002_chartrecord_charts_char_user_id_1adcde_idx_and_more.py b/vrobbler/apps/charts/migrations/0002_chartrecord_charts_char_user_id_1adcde_idx_and_more.py new file mode 100644 index 0000000..46ebc7c --- /dev/null +++ b/vrobbler/apps/charts/migrations/0002_chartrecord_charts_char_user_id_1adcde_idx_and_more.py @@ -0,0 +1,146 @@ +# Generated by Django 4.2.29 on 2026-06-12 16:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("charts", "0001_initial"), + ] + + operations = [ + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "album", "rank"], + name="charts_char_user_id_1adcde_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "track", "rank"], + name="charts_char_user_id_d18aab_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "video", "rank"], + name="charts_char_user_id_de9f0a_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "board_game", "rank"], + name="charts_char_user_id_d5d58f_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "book", "rank"], + name="charts_char_user_id_e877cf_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "food", "rank"], + name="charts_char_user_id_a0ad71_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "podcast", "rank"], + name="charts_char_user_id_846b80_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "trail", "rank"], + name="charts_char_user_id_54feba_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "album", "rank"], + name="charts_char_user_id_a3dc49_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "track", "rank"], + name="charts_char_user_id_4b01ab_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "video", "rank"], + name="charts_char_user_id_2ac9d2_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "board_game", "rank"], + name="charts_char_user_id_ba968a_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "book", "rank"], + name="charts_char_user_id_e66751_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "food", "rank"], + name="charts_char_user_id_d23f06_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "podcast", "rank"], + name="charts_char_user_id_be8122_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "week", "trail", "rank"], + name="charts_char_user_id_b94ea9_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "day", "artist", "rank"], + name="charts_char_user_id_406e0e_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "day", "album", "rank"], + name="charts_char_user_id_322b0d_idx", + ), + ), + migrations.AddIndex( + model_name="chartrecord", + index=models.Index( + fields=["user", "year", "month", "day", "tv_series", "rank"], + name="charts_char_user_id_aa44b7_idx", + ), + ), + ] diff --git a/vrobbler/apps/charts/models.py b/vrobbler/apps/charts/models.py index e1ed073..a84f35e 100644 --- a/vrobbler/apps/charts/models.py +++ b/vrobbler/apps/charts/models.py @@ -60,10 +60,29 @@ class ChartRecord(TimeStampedModel): models.Index(fields=["user", "year", "geo_location", "rank"]), models.Index(fields=["user", "year", "food", "rank"]), models.Index(fields=["user", "year", "book", "rank"]), - models.Index(fields=["user", "year", "week", "artist", "rank"]), - models.Index(fields=["user", "year", "week", "tv_series", "rank"]), models.Index(fields=["user", "year", "month", "artist", "rank"]), + models.Index(fields=["user", "year", "month", "album", "rank"]), + models.Index(fields=["user", "year", "month", "track", "rank"]), models.Index(fields=["user", "year", "month", "tv_series", "rank"]), + models.Index(fields=["user", "year", "month", "video", "rank"]), + models.Index(fields=["user", "year", "month", "board_game", "rank"]), + models.Index(fields=["user", "year", "month", "book", "rank"]), + models.Index(fields=["user", "year", "month", "food", "rank"]), + models.Index(fields=["user", "year", "month", "podcast", "rank"]), + models.Index(fields=["user", "year", "month", "trail", "rank"]), + models.Index(fields=["user", "year", "week", "artist", "rank"]), + models.Index(fields=["user", "year", "week", "album", "rank"]), + models.Index(fields=["user", "year", "week", "track", "rank"]), + models.Index(fields=["user", "year", "week", "tv_series", "rank"]), + models.Index(fields=["user", "year", "week", "video", "rank"]), + models.Index(fields=["user", "year", "week", "board_game", "rank"]), + models.Index(fields=["user", "year", "week", "book", "rank"]), + models.Index(fields=["user", "year", "week", "food", "rank"]), + models.Index(fields=["user", "year", "week", "podcast", "rank"]), + models.Index(fields=["user", "year", "week", "trail", "rank"]), + models.Index(fields=["user", "year", "month", "day", "artist", "rank"]), + models.Index(fields=["user", "year", "month", "day", "album", "rank"]), + models.Index(fields=["user", "year", "month", "day", "tv_series", "rank"]), ] @property diff --git a/vrobbler/apps/charts/views.py b/vrobbler/apps/charts/views.py index feed131..803f1e7 100644 --- a/vrobbler/apps/charts/views.py +++ b/vrobbler/apps/charts/views.py @@ -438,12 +438,14 @@ class ChartRecordView(TemplateView): return context def get_available_years(self, user): - return list( - ChartRecord.objects.filter(user=user) - .values_list("year", flat=True) - .distinct() - .order_by("-year") - ) + if not hasattr(self, "_available_years"): + self._available_years = list( + ChartRecord.objects.filter(user=user) + .values_list("year", flat=True) + .distinct() + .order_by("-year") + ) + return self._available_years def get_period_type(self): date_param = self.request.GET.get("date") diff --git a/vrobbler/apps/scrobbles/migrations/0097_scrobble_scrobbles_s_user_id_d367a7_idx.py b/vrobbler/apps/scrobbles/migrations/0097_scrobble_scrobbles_s_user_id_d367a7_idx.py new file mode 100644 index 0000000..374b879 --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0097_scrobble_scrobbles_s_user_id_d367a7_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.29 on 2026-06-12 16:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scrobbles", "0096_convert_book_page_data_to_dict"), + ] + + operations = [ + migrations.AddIndex( + model_name="scrobble", + index=models.Index( + fields=["user", "-timestamp"], name="scrobbles_s_user_id_d367a7_idx" + ), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 6f165a3..4390e5d 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -612,6 +612,30 @@ class EBirdCSVImport(BaseFileImportMixin): self.mark_finished() +TYPE_FK_PREFETCHES: dict[str, tuple[str, ...]] = { + "Video": ("video",), + "Track": ("track", "track__artist_fk"), + "PodcastEpisode": ("podcast_episode", "podcast_episode__podcast"), + "SportEvent": ("sport_event",), + "Book": ("book",), + "Paper": ("paper",), + "VideoGame": ("video_game",), + "BoardGame": ("board_game",), + "GeoLocation": ("geo_location",), + "Trail": ("trail",), + "Beer": ("beer",), + "Puzzle": ("puzzle",), + "Food": ("food",), + "Task": ("task",), + "WebPage": ("web_page",), + "LifeEvent": ("life_event",), + "Mood": ("mood",), + "BrickSet": ("brick_set",), + "Channel": ("channel",), + "BirdingLocation": ("birding_location",), +} + + class ScrobbleQuerySet(models.QuerySet): def with_related(self): return self.select_related("user").prefetch_related( @@ -638,6 +662,13 @@ class ScrobbleQuerySet(models.QuerySet): "birding_location", ) + def with_related_for_types(self, media_types: list[str]): + prefetches = [] + for t in media_types: + if t in TYPE_FK_PREFETCHES: + prefetches.extend(TYPE_FK_PREFETCHES[t]) + return self.select_related("user").prefetch_related(*prefetches) + class ShareViewLog(TimeStampedModel): scrobble = models.ForeignKey( @@ -778,6 +809,7 @@ class Scrobble(TimeStampedModel): "is_paused", ] ), + models.Index(fields=["user", "-timestamp"]), ] @classmethod @@ -810,11 +842,14 @@ class Scrobble(TimeStampedModel): @classmethod def as_dict_by_type(cls, scrobble_qs: models.QuerySet) -> dict: scrobbles_by_type = defaultdict(list) - scrobbles = ( - scrobble_qs.with_related() - if hasattr(scrobble_qs, "with_related") - else scrobble_qs - ) + + if hasattr(scrobble_qs, "with_related"): + media_types_present = list( + scrobble_qs.values_list("media_type", flat=True).distinct() + ) + scrobbles = scrobble_qs.with_related_for_types(media_types_present) + else: + scrobbles = scrobble_qs for scrobble in scrobbles: scrobbles_by_type[scrobble.media_type].append(scrobble) @@ -829,7 +864,7 @@ class Scrobble(TimeStampedModel): # Remove any locations without titles if "GeoLocation" in scrobbles_by_type.keys(): - for loc_scrobble in scrobbles_by_type["GeoLocation"]: + for loc_scrobble in list(scrobbles_by_type["GeoLocation"]): if not loc_scrobble.media_obj.title: scrobbles_by_type["GeoLocation"].remove(loc_scrobble) scrobbles_by_type["GeoLocation_count"] -= 1