diff --git a/PROJECT.org b/PROJECT.org index 26c7344..4b88d1d 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/22] :vrobbler:project:personal: +* Backlog [0/23] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -604,6 +604,11 @@ independent of the email flow it was originally creatdd for ** TODO [#B] Is there way to create unique slugs for media instances :media_types: +** STRT [#A] Tighten up the speed of startup and first request :perf: +:PROPERTIES: +:ID: 9ee8834c-6be2-d04b-df6d-56375504083f +:END: + * Version 55.3 [3/3] ** DONE [#C] =alt_names= feature for artists (commented out / dead code) :music:dead-code: :PROPERTIES: diff --git a/tests/test_context_processors.py b/tests/test_context_processors.py index 76b0835..931ef57 100644 --- a/tests/test_context_processors.py +++ b/tests/test_context_processors.py @@ -4,9 +4,15 @@ from unittest.mock import MagicMock, patch import pytest +from vrobbler import context_processors from vrobbler.context_processors import version_info +@pytest.fixture(autouse=True) +def reset_git_cache(): + context_processors._GIT_COMMIT = None + + @pytest.fixture def mock_request(): return MagicMock() diff --git a/vrobbler/apps/charts/migrations/0003_chartrecord_unique_chart_artist_period_and_more.py b/vrobbler/apps/charts/migrations/0003_chartrecord_unique_chart_artist_period_and_more.py new file mode 100644 index 0000000..4d05bfb --- /dev/null +++ b/vrobbler/apps/charts/migrations/0003_chartrecord_unique_chart_artist_period_and_more.py @@ -0,0 +1,109 @@ +# Generated by Django 4.2.29 on 2026-06-19 16:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("charts", "0002_chartrecord_charts_char_user_id_1adcde_idx_and_more"), + ] + + operations = [ + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("artist__isnull", False)), + fields=("user", "year", "month", "week", "day", "artist"), + name="unique_chart_artist_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("album__isnull", False)), + fields=("user", "year", "month", "week", "day", "album"), + name="unique_chart_album_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("track__isnull", False)), + fields=("user", "year", "month", "week", "day", "track"), + name="unique_chart_track_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("tv_series__isnull", False)), + fields=("user", "year", "month", "week", "day", "tv_series"), + name="unique_chart_tv_series_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("video__isnull", False)), + fields=("user", "year", "month", "week", "day", "video"), + name="unique_chart_video_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("podcast__isnull", False)), + fields=("user", "year", "month", "week", "day", "podcast"), + name="unique_chart_podcast_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("podcast_episode__isnull", False)), + fields=("user", "year", "month", "week", "day", "podcast_episode"), + name="unique_chart_podcast_episode_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("board_game__isnull", False)), + fields=("user", "year", "month", "week", "day", "board_game"), + name="unique_chart_board_game_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("trail__isnull", False)), + fields=("user", "year", "month", "week", "day", "trail"), + name="unique_chart_trail_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("geo_location__isnull", False)), + fields=("user", "year", "month", "week", "day", "geo_location"), + name="unique_chart_geo_location_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("food__isnull", False)), + fields=("user", "year", "month", "week", "day", "food"), + name="unique_chart_food_period", + ), + ), + migrations.AddConstraint( + model_name="chartrecord", + constraint=models.UniqueConstraint( + condition=models.Q(("book__isnull", False)), + fields=("user", "year", "month", "week", "day", "book"), + name="unique_chart_book_period", + ), + ), + ] diff --git a/vrobbler/apps/charts/models.py b/vrobbler/apps/charts/models.py index a84f35e..a2012ca 100644 --- a/vrobbler/apps/charts/models.py +++ b/vrobbler/apps/charts/models.py @@ -2,6 +2,7 @@ import calendar from django.contrib.auth import get_user_model from django.db import models +from django.db.models import Q from django.urls import reverse from django_extensions.db.models import TimeStampedModel @@ -84,6 +85,68 @@ class ChartRecord(TimeStampedModel): models.Index(fields=["user", "year", "month", "day", "album", "rank"]), models.Index(fields=["user", "year", "month", "day", "tv_series", "rank"]), ] + constraints = [ + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "artist"], + condition=Q(artist__isnull=False), + name="unique_chart_artist_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "album"], + condition=Q(album__isnull=False), + name="unique_chart_album_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "track"], + condition=Q(track__isnull=False), + name="unique_chart_track_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "tv_series"], + condition=Q(tv_series__isnull=False), + name="unique_chart_tv_series_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "video"], + condition=Q(video__isnull=False), + name="unique_chart_video_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "podcast"], + condition=Q(podcast__isnull=False), + name="unique_chart_podcast_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "podcast_episode"], + condition=Q(podcast_episode__isnull=False), + name="unique_chart_podcast_episode_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "board_game"], + condition=Q(board_game__isnull=False), + name="unique_chart_board_game_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "trail"], + condition=Q(trail__isnull=False), + name="unique_chart_trail_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "geo_location"], + condition=Q(geo_location__isnull=False), + name="unique_chart_geo_location_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "food"], + condition=Q(food__isnull=False), + name="unique_chart_food_period", + ), + models.UniqueConstraint( + fields=["user", "year", "month", "week", "day", "book"], + condition=Q(book__isnull=False), + name="unique_chart_book_period", + ), + ] @property def media_obj(self): diff --git a/vrobbler/apps/charts/utils.py b/vrobbler/apps/charts/utils.py index 6db9eb6..9c15334 100644 --- a/vrobbler/apps/charts/utils.py +++ b/vrobbler/apps/charts/utils.py @@ -6,6 +6,7 @@ from typing import Optional import pytz from django.apps import apps from django.conf import settings +from django.db import transaction from django.db.models import Count, Q from django.utils import timezone @@ -186,60 +187,64 @@ def build_charts( ranks = {count: rank for rank, count in enumerate(unique_counts, start=1)} media_field = f"{media_type}_id" - records_to_create = [] - records_to_update = [] - existing = ChartRecord.objects.filter( - period_filter, user=user, **{media_field + "__isnull": False} - ) - existing_by_media_id = {getattr(r, media_field): r for r in existing} - found_media_ids = set() + with transaction.atomic(): + records_to_create = [] + records_to_update = [] - for result in results: - media_id = result[config["values"]] - if media_id is None: - continue - - found_media_ids.add(media_id) - - chart_record_data = { - "user_id": user.id, - "year": year, - "month": month, - "week": week, - "day": day, - "rank": ranks[result["scrobble_count"]], - "count": result["scrobble_count"], - } - chart_record_data[media_field] = media_id - - if media_id in existing_by_media_id: - existing_record = existing_by_media_id[media_id] - existing_record.rank = chart_record_data["rank"] - existing_record.count = chart_record_data["count"] - records_to_update.append(existing_record) - else: - records_to_create.append(ChartRecord(**chart_record_data)) - - ids_to_delete = [ - r.id for r in existing if getattr(r, media_field) not in found_media_ids - ] - if ids_to_delete: - ChartRecord.objects.filter(id__in=ids_to_delete).delete() - - if records_to_update: - ChartRecord.objects.bulk_update( - records_to_update, ["rank", "count"], batch_size=500 + existing = ChartRecord.objects.select_for_update().filter( + period_filter, user=user, **{media_field + "__isnull": False} ) + existing_by_media_id = {getattr(r, media_field): r for r in existing} + found_media_ids = set() - if records_to_create: - ChartRecord.objects.bulk_create(records_to_create, batch_size=500) + for result in results: + media_id = result[config["values"]] + if media_id is None: + continue - logger.info( - f"Built {len(records_to_create)} new, {len(records_to_update)} updated " - f"chart records for {media_type}, period " - f"{year}-{month or ''}-{week or ''}-{day or ''} for {user}" - ) + found_media_ids.add(media_id) + + chart_record_data = { + "user_id": user.id, + "year": year, + "month": month, + "week": week, + "day": day, + "rank": ranks[result["scrobble_count"]], + "count": result["scrobble_count"], + } + chart_record_data[media_field] = media_id + + if media_id in existing_by_media_id: + existing_record = existing_by_media_id[media_id] + existing_record.rank = chart_record_data["rank"] + existing_record.count = chart_record_data["count"] + records_to_update.append(existing_record) + else: + records_to_create.append(ChartRecord(**chart_record_data)) + + ids_to_delete = [ + r.id + for r in existing + if getattr(r, media_field) not in found_media_ids + ] + if ids_to_delete: + ChartRecord.objects.filter(id__in=ids_to_delete).delete() + + if records_to_update: + ChartRecord.objects.bulk_update( + records_to_update, ["rank", "count"], batch_size=500 + ) + + if records_to_create: + ChartRecord.objects.bulk_create(records_to_create, batch_size=500) + + logger.info( + f"Built {len(records_to_create)} new, {len(records_to_update)} updated " + f"chart records for {media_type}, period " + f"{year}-{month or ''}-{week or ''}-{day or ''} for {user}" + ) def build_yesterdays_charts(user, media_types: Optional[list] = None) -> None: diff --git a/vrobbler/apps/music/context_processors.py b/vrobbler/apps/music/context_processors.py index 3db98e0..ab2ef9a 100644 --- a/vrobbler/apps/music/context_processors.py +++ b/vrobbler/apps/music/context_processors.py @@ -1,8 +1,22 @@ +from django.core.cache import cache + from music.models import Artist, Album +CACHE_TTL = 300 + def music_lists(request): + artist_list = cache.get("music_lists_artist_list") + if artist_list is None: + artist_list = list(Artist.objects.all().only("id", "name")) + cache.set("music_lists_artist_list", artist_list, CACHE_TTL) + + album_list = cache.get("music_lists_album_list") + if album_list is None: + album_list = list(Album.objects.all().only("id", "name")) + cache.set("music_lists_album_list", album_list, CACHE_TTL) + return { - "artist_list": Artist.objects.all(), - "album_list": Album.objects.all(), + "artist_list": artist_list, + "album_list": album_list, } diff --git a/vrobbler/apps/scrobbles/context_processors.py b/vrobbler/apps/scrobbles/context_processors.py index 2247f44..fa9a642 100644 --- a/vrobbler/apps/scrobbles/context_processors.py +++ b/vrobbler/apps/scrobbles/context_processors.py @@ -1,4 +1,5 @@ import pytz +from django.core.cache import cache from django.utils import timezone from scrobbles.constants import EXCLUDE_FROM_NOW_PLAYING @@ -19,6 +20,8 @@ MONTH_COLORS = [ "#db7a7a", # Dec ] +CACHE_TTL = 60 + def month_color(request): from datetime import date @@ -27,15 +30,25 @@ def month_color(request): def now_playing(request): user = request.user - now = timezone.now() if not user.is_authenticated: return {} - return { - "now_playing_list": Scrobble.objects.filter( - in_progress=True, - is_paused=False, - user=user, - ).exclude( - media_type__in=EXCLUDE_FROM_NOW_PLAYING, + + cache_key = f"now_playing_list_{user.id}" + now_playing_list = cache.get(cache_key) + if now_playing_list is None: + now_playing_list = list( + Scrobble.objects.filter( + in_progress=True, + is_paused=False, + user=user, + ) + .exclude( + media_type__in=EXCLUDE_FROM_NOW_PLAYING, + ) + .select_related("track", "video", "podcast_episode") ) + cache.set(cache_key, now_playing_list, CACHE_TTL) + + return { + "now_playing_list": now_playing_list, } diff --git a/vrobbler/apps/scrobbles/signals.py b/vrobbler/apps/scrobbles/signals.py index 9a1f0d4..0593c22 100644 --- a/vrobbler/apps/scrobbles/signals.py +++ b/vrobbler/apps/scrobbles/signals.py @@ -1,5 +1,6 @@ import logging +from django.core.cache import cache from django.db.models.signals import post_delete, post_save from django.dispatch import receiver from django.utils import timezone @@ -52,6 +53,11 @@ def _update_charts_for_timestamp(user, ts): if ts is None: return + lock_key = f"chart_update_{user.id}" + if not cache.add(lock_key, "locked", timeout=30): + logger.info(f"Chart update already queued for user {user.id}, skipping") + return + if timezone.is_naive(ts): ts = timezone.make_aware(ts) diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index 5d04bdf..a245129 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -12,6 +12,7 @@ from charts.utils import ( from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model +from django.core.cache import cache from django.db import models from django.utils import timezone @@ -241,6 +242,11 @@ def update_charts_for_timestamp(user_id, year, month, day, week): logger.error(f"User with id {user_id} not found") return + lock_key = f"chart_update_running_{user_id}" + if not cache.add(lock_key, "locked", timeout=300): + logger.info(f"Chart update already running for user {user_id}, skipping") + return + try: build_daily_charts(user, year, month, day, CHARTABLE_MEDIA_TYPES) build_weekly_charts(user, year, week, CHARTABLE_MEDIA_TYPES) @@ -250,6 +256,8 @@ def update_charts_for_timestamp(user_id, year, month, day, week): logger.info(f"[charts] Updated charts for {user} on {date_str}") except Exception as e: logger.error(f"[charts] Failed to update charts: {e}") + finally: + cache.delete(lock_key) @shared_task diff --git a/vrobbler/apps/videos/context_processors.py b/vrobbler/apps/videos/context_processors.py index 34be4aa..3f6b3b5 100644 --- a/vrobbler/apps/videos/context_processors.py +++ b/vrobbler/apps/videos/context_processors.py @@ -1,8 +1,24 @@ +from django.core.cache import cache + from videos.models import Video, Series +CACHE_TTL = 300 + def video_lists(request): + movie_list = cache.get("video_lists_movie_list") + if movie_list is None: + movie_list = list( + Video.objects.filter(video_type=Video.VideoType.MOVIE).only("id", "title") + ) + cache.set("video_lists_movie_list", movie_list, CACHE_TTL) + + series_list = cache.get("video_lists_series_list") + if series_list is None: + series_list = list(Series.objects.all().only("id", "name")) + cache.set("video_lists_series_list", series_list, CACHE_TTL) + return { - "movie_list": Video.objects.filter(video_type=Video.VideoType.MOVIE), - "series_list": Series.objects.all(), + "movie_list": movie_list, + "series_list": series_list, } diff --git a/vrobbler/context_processors.py b/vrobbler/context_processors.py index 79bbb0b..5d28d3f 100644 --- a/vrobbler/context_processors.py +++ b/vrobbler/context_processors.py @@ -4,46 +4,55 @@ from importlib.metadata import version as get_version from pathlib import Path +_GIT_COMMIT = None + + def version_info(request): + global _GIT_COMMIT + try: app_version = get_version("vrobbler") except Exception: app_version = "unknown" commit = os.environ.get("VROBBLER_COMMIT") - if not commit: - # Try to import from _commit.py module first + if commit: + return {"app_version": app_version, "git_commit": commit} + + if _GIT_COMMIT is not None: + return {"app_version": app_version, "git_commit": _GIT_COMMIT} + + try: + from vrobbler._commit import commit as _commit + except ImportError: + pass + else: + if _commit and _commit != "unknown": + _GIT_COMMIT = _commit + return {"app_version": app_version, "git_commit": _GIT_COMMIT} + + commit_file = Path("/var/lib/vrobbler/commit.txt") + if commit_file.exists(): try: - from vrobbler._commit import commit as _commit - except ImportError: + commit = commit_file.read_text().strip() + if commit: + _GIT_COMMIT = commit + return {"app_version": app_version, "git_commit": _GIT_COMMIT} + except OSError: pass - else: - if _commit and _commit != "unknown": - return {"app_version": app_version, "git_commit": _commit} - # Try to read from commit file (written during deploy) - commit_file = Path("/var/lib/vrobbler/commit.txt") - if commit_file.exists(): - try: - commit = commit_file.read_text().strip() - if commit: - return {"app_version": app_version, "git_commit": commit} - except OSError: - pass - - # Fall back to git command - PROJECT_ROOT = Path(__file__).resolve().parent.parent - try: - commit = ( - subprocess.check_output( - ["git", "rev-parse", "--short", "HEAD"], - cwd=PROJECT_ROOT, - stderr=subprocess.DEVNULL, - ) - .decode("utf-8") - .strip() + PROJECT_ROOT = Path(__file__).resolve().parent.parent + try: + _GIT_COMMIT = ( + subprocess.check_output( + ["git", "rev-parse", "--short", "HEAD"], + cwd=PROJECT_ROOT, + stderr=subprocess.DEVNULL, ) - except (subprocess.SubprocessError, FileNotFoundError): - commit = "unknown" + .decode("utf-8") + .strip() + ) + except (subprocess.SubprocessError, FileNotFoundError): + _GIT_COMMIT = "unknown" - return {"app_version": app_version, "git_commit": commit} + return {"app_version": app_version, "git_commit": _GIT_COMMIT} diff --git a/vrobbler/settings.py b/vrobbler/settings.py index aad6fc8..7fad0e0 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -361,11 +361,11 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ # -from storages.backends import s3boto3 - USE_S3_STORAGE = os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY if USE_S3_STORAGE: + from storages.backends import s3boto3 + AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", "") AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "") AWS_S3_ACCESS_KEY_ID = os.getenv("AWS_S3_ACCESS_KEY_ID")