[perf] Add caching and lock protections

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [0/22] :vrobbler:project:personal:
+* Backlog [0/23] :vrobbler:project:personal:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles:
 :PROPERTIES:
 :ID:       702462cf-d54b-48c6-8a7c-78b8de751deb
@@ -604,6 +604,11 @@ independent of the email flow it was originally creatdd for
 
 ** TODO [#B] Is there way to create unique slugs for media instances :media_types:
 
+** STRT [#A] Tighten up the speed of startup and first request :perf:
+:PROPERTIES:
+:ID:       9ee8834c-6be2-d04b-df6d-56375504083f
+:END:
+
 * Version 55.3 [3/3]
 ** DONE [#C] =alt_names= feature for artists (commented out / dead code) :music:dead-code:
 :PROPERTIES:

tests/test_context_processors.py

@@ -4,9 +4,15 @@ from unittest.mock import MagicMock, patch
 
 import pytest
 
+from vrobbler import context_processors
 from vrobbler.context_processors import version_info
 
 
+@pytest.fixture(autouse=True)
+def reset_git_cache():
+    context_processors._GIT_COMMIT = None
+
+
 @pytest.fixture
 def mock_request():
     return MagicMock()

vrobbler/apps/charts/migrations/0003_chartrecord_unique_chart_artist_period_and_more.py

@@ -0,0 +1,109 @@
+# Generated by Django 4.2.29 on 2026-06-19 16:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("charts", "0002_chartrecord_charts_char_user_id_1adcde_idx_and_more"),
+    ]
+
+    operations = [
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("artist__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "artist"),
+                name="unique_chart_artist_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("album__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "album"),
+                name="unique_chart_album_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("track__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "track"),
+                name="unique_chart_track_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("tv_series__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "tv_series"),
+                name="unique_chart_tv_series_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("video__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "video"),
+                name="unique_chart_video_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("podcast__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "podcast"),
+                name="unique_chart_podcast_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("podcast_episode__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "podcast_episode"),
+                name="unique_chart_podcast_episode_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("board_game__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "board_game"),
+                name="unique_chart_board_game_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("trail__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "trail"),
+                name="unique_chart_trail_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("geo_location__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "geo_location"),
+                name="unique_chart_geo_location_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("food__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "food"),
+                name="unique_chart_food_period",
+            ),
+        ),
+        migrations.AddConstraint(
+            model_name="chartrecord",
+            constraint=models.UniqueConstraint(
+                condition=models.Q(("book__isnull", False)),
+                fields=("user", "year", "month", "week", "day", "book"),
+                name="unique_chart_book_period",
+            ),
+        ),
+    ]

vrobbler/apps/charts/models.py

@@ -2,6 +2,7 @@ import calendar
 
 from django.contrib.auth import get_user_model
 from django.db import models
+from django.db.models import Q
 from django.urls import reverse
 from django_extensions.db.models import TimeStampedModel
 
@@ -84,6 +85,68 @@ class ChartRecord(TimeStampedModel):
             models.Index(fields=["user", "year", "month", "day", "album", "rank"]),
             models.Index(fields=["user", "year", "month", "day", "tv_series", "rank"]),
         ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "artist"],
+                condition=Q(artist__isnull=False),
+                name="unique_chart_artist_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "album"],
+                condition=Q(album__isnull=False),
+                name="unique_chart_album_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "track"],
+                condition=Q(track__isnull=False),
+                name="unique_chart_track_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "tv_series"],
+                condition=Q(tv_series__isnull=False),
+                name="unique_chart_tv_series_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "video"],
+                condition=Q(video__isnull=False),
+                name="unique_chart_video_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "podcast"],
+                condition=Q(podcast__isnull=False),
+                name="unique_chart_podcast_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "podcast_episode"],
+                condition=Q(podcast_episode__isnull=False),
+                name="unique_chart_podcast_episode_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "board_game"],
+                condition=Q(board_game__isnull=False),
+                name="unique_chart_board_game_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "trail"],
+                condition=Q(trail__isnull=False),
+                name="unique_chart_trail_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "geo_location"],
+                condition=Q(geo_location__isnull=False),
+                name="unique_chart_geo_location_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "food"],
+                condition=Q(food__isnull=False),
+                name="unique_chart_food_period",
+            ),
+            models.UniqueConstraint(
+                fields=["user", "year", "month", "week", "day", "book"],
+                condition=Q(book__isnull=False),
+                name="unique_chart_book_period",
+            ),
+        ]
 
     @property
     def media_obj(self):

vrobbler/apps/charts/utils.py

@@ -6,6 +6,7 @@ from typing import Optional
 import pytz
 from django.apps import apps
 from django.conf import settings
+from django.db import transaction
 from django.db.models import Count, Q
 from django.utils import timezone
 
@@ -186,60 +187,64 @@ def build_charts(
         ranks = {count: rank for rank, count in enumerate(unique_counts, start=1)}
 
         media_field = f"{media_type}_id"
-        records_to_create = []
-        records_to_update = []
 
-        existing = ChartRecord.objects.filter(
+        with transaction.atomic():
-            period_filter, user=user, **{media_field + "__isnull": False}
+            records_to_create = []
-        )
+            records_to_update = []
-        existing_by_media_id = {getattr(r, media_field): r for r in existing}
-        found_media_ids = set()
 
-        for result in results:
+            existing = ChartRecord.objects.select_for_update().filter(
-            media_id = result[config["values"]]
+                period_filter, user=user, **{media_field + "__isnull": False}
-            if media_id is None:
-                continue
-
-            found_media_ids.add(media_id)
-
-            chart_record_data = {
-                "user_id": user.id,
-                "year": year,
-                "month": month,
-                "week": week,
-                "day": day,
-                "rank": ranks[result["scrobble_count"]],
-                "count": result["scrobble_count"],
-            }
-            chart_record_data[media_field] = media_id
-
-            if media_id in existing_by_media_id:
-                existing_record = existing_by_media_id[media_id]
-                existing_record.rank = chart_record_data["rank"]
-                existing_record.count = chart_record_data["count"]
-                records_to_update.append(existing_record)
-            else:
-                records_to_create.append(ChartRecord(**chart_record_data))
-
-        ids_to_delete = [
-            r.id for r in existing if getattr(r, media_field) not in found_media_ids
-        ]
-        if ids_to_delete:
-            ChartRecord.objects.filter(id__in=ids_to_delete).delete()
-
-        if records_to_update:
-            ChartRecord.objects.bulk_update(
-                records_to_update, ["rank", "count"], batch_size=500
             )
+            existing_by_media_id = {getattr(r, media_field): r for r in existing}
+            found_media_ids = set()
 
-        if records_to_create:
+            for result in results:
-            ChartRecord.objects.bulk_create(records_to_create, batch_size=500)
+                media_id = result[config["values"]]
+                if media_id is None:
+                    continue
 
-        logger.info(
+                found_media_ids.add(media_id)
-            f"Built {len(records_to_create)} new, {len(records_to_update)} updated "
+
-            f"chart records for {media_type}, period "
+                chart_record_data = {
-            f"{year}-{month or ''}-{week or ''}-{day or ''} for {user}"
+                    "user_id": user.id,
-        )
+                    "year": year,
+                    "month": month,
+                    "week": week,
+                    "day": day,
+                    "rank": ranks[result["scrobble_count"]],
+                    "count": result["scrobble_count"],
+                }
+                chart_record_data[media_field] = media_id
+
+                if media_id in existing_by_media_id:
+                    existing_record = existing_by_media_id[media_id]
+                    existing_record.rank = chart_record_data["rank"]
+                    existing_record.count = chart_record_data["count"]
+                    records_to_update.append(existing_record)
+                else:
+                    records_to_create.append(ChartRecord(**chart_record_data))
+
+            ids_to_delete = [
+                r.id
+                for r in existing
+                if getattr(r, media_field) not in found_media_ids
+            ]
+            if ids_to_delete:
+                ChartRecord.objects.filter(id__in=ids_to_delete).delete()
+
+            if records_to_update:
+                ChartRecord.objects.bulk_update(
+                    records_to_update, ["rank", "count"], batch_size=500
+                )
+
+            if records_to_create:
+                ChartRecord.objects.bulk_create(records_to_create, batch_size=500)
+
+            logger.info(
+                f"Built {len(records_to_create)} new, {len(records_to_update)} updated "
+                f"chart records for {media_type}, period "
+                f"{year}-{month or ''}-{week or ''}-{day or ''} for {user}"
+            )
 
 
 def build_yesterdays_charts(user, media_types: Optional[list] = None) -> None:

vrobbler/apps/music/context_processors.py

@@ -1,8 +1,22 @@
+from django.core.cache import cache
+
 from music.models import Artist, Album
 
+CACHE_TTL = 300
+
 
 def music_lists(request):
+    artist_list = cache.get("music_lists_artist_list")
+    if artist_list is None:
+        artist_list = list(Artist.objects.all().only("id", "name"))
+        cache.set("music_lists_artist_list", artist_list, CACHE_TTL)
+
+    album_list = cache.get("music_lists_album_list")
+    if album_list is None:
+        album_list = list(Album.objects.all().only("id", "name"))
+        cache.set("music_lists_album_list", album_list, CACHE_TTL)
+
     return {
-        "artist_list": Artist.objects.all(),
+        "artist_list": artist_list,
-        "album_list": Album.objects.all(),
+        "album_list": album_list,
     }

vrobbler/apps/scrobbles/context_processors.py

@@ -1,4 +1,5 @@
 import pytz
+from django.core.cache import cache
 from django.utils import timezone
 
 from scrobbles.constants import EXCLUDE_FROM_NOW_PLAYING
@@ -19,6 +20,8 @@ MONTH_COLORS = [
     "#db7a7a",  # Dec
 ]
 
+CACHE_TTL = 60
+
 
 def month_color(request):
     from datetime import date
@@ -27,15 +30,25 @@ def month_color(request):
 
 def now_playing(request):
     user = request.user
-    now = timezone.now()
     if not user.is_authenticated:
         return {}
-    return {
+
-        "now_playing_list": Scrobble.objects.filter(
+    cache_key = f"now_playing_list_{user.id}"
-            in_progress=True,
+    now_playing_list = cache.get(cache_key)
-            is_paused=False,
+    if now_playing_list is None:
-            user=user,
+        now_playing_list = list(
-        ).exclude(
+            Scrobble.objects.filter(
-            media_type__in=EXCLUDE_FROM_NOW_PLAYING,
+                in_progress=True,
+                is_paused=False,
+                user=user,
+            )
+            .exclude(
+                media_type__in=EXCLUDE_FROM_NOW_PLAYING,
+            )
+            .select_related("track", "video", "podcast_episode")
         )
+        cache.set(cache_key, now_playing_list, CACHE_TTL)
+
+    return {
+        "now_playing_list": now_playing_list,
     }

vrobbler/apps/scrobbles/signals.py

@@ -1,5 +1,6 @@
 import logging
 
+from django.core.cache import cache
 from django.db.models.signals import post_delete, post_save
 from django.dispatch import receiver
 from django.utils import timezone
@@ -52,6 +53,11 @@ def _update_charts_for_timestamp(user, ts):
     if ts is None:
         return
 
+    lock_key = f"chart_update_{user.id}"
+    if not cache.add(lock_key, "locked", timeout=30):
+        logger.info(f"Chart update already queued for user {user.id}, skipping")
+        return
+
     if timezone.is_naive(ts):
         ts = timezone.make_aware(ts)
 

vrobbler/apps/scrobbles/tasks.py

@@ -12,6 +12,7 @@ from charts.utils import (
 from django.apps import apps
 from django.conf import settings
 from django.contrib.auth import get_user_model
+from django.core.cache import cache
 from django.db import models
 from django.utils import timezone
 
@@ -241,6 +242,11 @@ def update_charts_for_timestamp(user_id, year, month, day, week):
         logger.error(f"User with id {user_id} not found")
         return
 
+    lock_key = f"chart_update_running_{user_id}"
+    if not cache.add(lock_key, "locked", timeout=300):
+        logger.info(f"Chart update already running for user {user_id}, skipping")
+        return
+
     try:
         build_daily_charts(user, year, month, day, CHARTABLE_MEDIA_TYPES)
         build_weekly_charts(user, year, week, CHARTABLE_MEDIA_TYPES)
@@ -250,6 +256,8 @@ def update_charts_for_timestamp(user_id, year, month, day, week):
         logger.info(f"[charts] Updated charts for {user} on {date_str}")
     except Exception as e:
         logger.error(f"[charts] Failed to update charts: {e}")
+    finally:
+        cache.delete(lock_key)
 
 
 @shared_task

vrobbler/apps/videos/context_processors.py

@@ -1,8 +1,24 @@
+from django.core.cache import cache
+
 from videos.models import Video, Series
 
+CACHE_TTL = 300
+
 
 def video_lists(request):
+    movie_list = cache.get("video_lists_movie_list")
+    if movie_list is None:
+        movie_list = list(
+            Video.objects.filter(video_type=Video.VideoType.MOVIE).only("id", "title")
+        )
+        cache.set("video_lists_movie_list", movie_list, CACHE_TTL)
+
+    series_list = cache.get("video_lists_series_list")
+    if series_list is None:
+        series_list = list(Series.objects.all().only("id", "name"))
+        cache.set("video_lists_series_list", series_list, CACHE_TTL)
+
     return {
-        "movie_list": Video.objects.filter(video_type=Video.VideoType.MOVIE),
+        "movie_list": movie_list,
-        "series_list": Series.objects.all(),
+        "series_list": series_list,
     }

vrobbler/context_processors.py

@@ -4,46 +4,55 @@ from importlib.metadata import version as get_version
 from pathlib import Path
 
 
+_GIT_COMMIT = None
+
+
 def version_info(request):
+    global _GIT_COMMIT
+
     try:
         app_version = get_version("vrobbler")
     except Exception:
         app_version = "unknown"
 
     commit = os.environ.get("VROBBLER_COMMIT")
-    if not commit:
+    if commit:
-        # Try to import from _commit.py module first
+        return {"app_version": app_version, "git_commit": commit}
+
+    if _GIT_COMMIT is not None:
+        return {"app_version": app_version, "git_commit": _GIT_COMMIT}
+
+    try:
+        from vrobbler._commit import commit as _commit
+    except ImportError:
+        pass
+    else:
+        if _commit and _commit != "unknown":
+            _GIT_COMMIT = _commit
+            return {"app_version": app_version, "git_commit": _GIT_COMMIT}
+
+    commit_file = Path("/var/lib/vrobbler/commit.txt")
+    if commit_file.exists():
         try:
-            from vrobbler._commit import commit as _commit
+            commit = commit_file.read_text().strip()
-        except ImportError:
+            if commit:
+                _GIT_COMMIT = commit
+                return {"app_version": app_version, "git_commit": _GIT_COMMIT}
+        except OSError:
             pass
-        else:
-            if _commit and _commit != "unknown":
-                return {"app_version": app_version, "git_commit": _commit}
 
-        # Try to read from commit file (written during deploy)
+    PROJECT_ROOT = Path(__file__).resolve().parent.parent
-        commit_file = Path("/var/lib/vrobbler/commit.txt")
+    try:
-        if commit_file.exists():
+        _GIT_COMMIT = (
-            try:
+            subprocess.check_output(
-                commit = commit_file.read_text().strip()
+                ["git", "rev-parse", "--short", "HEAD"],
-                if commit:
+                cwd=PROJECT_ROOT,
-                    return {"app_version": app_version, "git_commit": commit}
+                stderr=subprocess.DEVNULL,
-            except OSError:
-                pass
-
-        # Fall back to git command
-        PROJECT_ROOT = Path(__file__).resolve().parent.parent
-        try:
-            commit = (
-                subprocess.check_output(
-                    ["git", "rev-parse", "--short", "HEAD"],
-                    cwd=PROJECT_ROOT,
-                    stderr=subprocess.DEVNULL,
-                )
-                .decode("utf-8")
-                .strip()
             )
-        except (subprocess.SubprocessError, FileNotFoundError):
+            .decode("utf-8")
-            commit = "unknown"
+            .strip()
+        )
+    except (subprocess.SubprocessError, FileNotFoundError):
+        _GIT_COMMIT = "unknown"
 
-    return {"app_version": app_version, "git_commit": commit}
+    return {"app_version": app_version, "git_commit": _GIT_COMMIT}

vrobbler/settings.py

@@ -361,11 +361,11 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 #
-from storages.backends import s3boto3
-
 USE_S3_STORAGE = os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY
 
 if USE_S3_STORAGE:
+    from storages.backends import s3boto3
+
     AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", "")
     AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "")
     AWS_S3_ACCESS_KEY_ID = os.getenv("AWS_S3_ACCESS_KEY_ID")