secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[charts] Fix chart building

Browse Source
This commit is contained in:
Colin Powell
2026-03-22 12:59:54 -04:00
parent 02d13b5a99
commit 83d89001a7
22 changed files with 291 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
vrobbler/apps/boardgames/views.py
View File

@ -3,7 +3,11 @@ from django.utils import timezone
from django.views import generic
from boardgames.models import BoardGame, BoardGameDesigner, BoardGamePublisher
from scrobbles.models import Scrobble
from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView
from scrobbles.views import (
ChartContextMixin,
ScrobbleableListView,
ScrobbleableDetailView,
)
class BoardGameListView(ScrobbleableListView):
@ -64,7 +68,7 @@ class BoardGameListView(ScrobbleableListView):
return context_data
class BoardGameDetailView(ScrobbleableDetailView):
class BoardGameDetailView(ScrobbleableDetailView, ChartContextMixin):
model = BoardGame

6
vrobbler/apps/charts/admin.py
View File

@ -22,12 +22,6 @@ class ChartRecordAdmin(admin.ModelAdmin):
"month",
"week",
"day",
"tv_series",
"podcast",
"board_game",
"book",
"food",
"trail",
)
ordering = ("-created",)
search_fields = (

64
vrobbler/apps/charts/management/commands/build_charts.py
View File

@ -1,11 +1,58 @@
import logging
from charts.utils import build_all_historical_charts, rebuild_all_charts
from charts.utils import (
build_all_historical_charts,
build_charts_since,
rebuild_all_charts,
)
from django.core.management.base import BaseCommand
logger = logging.getLogger(__name__)
def get_chart_counts():
from charts.models import ChartRecord
daily = ChartRecord.objects.filter(day__isnull=False).count()
weekly = ChartRecord.objects.filter(week__isnull=False, day__isnull=True).count()
monthly = ChartRecord.objects.filter(
month__isnull=False, week__isnull=True, day__isnull=True
).count()
yearly = ChartRecord.objects.filter(
month__isnull=True, week__isnull=True, day__isnull=True
).count()
total = ChartRecord.objects.count()
return {
"daily": daily,
"weekly": weekly,
"monthly": monthly,
"yearly": yearly,
"total": total,
}
def print_chart_summary(before=None):
counts = get_chart_counts()
if before:
diff = {k: counts[k] - before[k] for k in counts}
print(
f" Daily: {counts['daily']} ({diff['daily']:+d}), "
f"Weekly: {counts['weekly']} ({diff['weekly']:+d}), "
f"Monthly: {counts['monthly']} ({diff['monthly']:+d}), "
f"Yearly: {counts['yearly']} ({diff['yearly']:+d}), "
f"Total: {counts['total']} ({diff['total']:+d})"
)
else:
print(
f" Daily: {counts['daily']}, "
f"Weekly: {counts['weekly']}, "
f"Monthly: {counts['monthly']}, "
f"Yearly: {counts['yearly']}, "
f"Total: {counts['total']}"
)
class Command(BaseCommand):
help = "Build missing charts for all users"
@ -38,22 +85,29 @@ class Command(BaseCommand):
)
return
users = [user]
action = "Rebuilding" if full_rebuild else "Building charts for"
action = "Full rebuild for" if full_rebuild else "Progressive build for"
self.stdout.write(f"{action} user: {user}")
else:
users = User.objects.all()
action = "Rebuilding" if full_rebuild else "Building charts for"
action = "Full rebuild for" if full_rebuild else "Progressive build for"
self.stdout.write(f"{action} charts for {users.count()} users...")
before_counts = get_chart_counts()
self.stdout.write("Chart counts before:")
print_chart_summary()
for user in users:
try:
logger.info(f"Building all historical charts for {user}")
logger.info(f"Building charts for {user}")
if full_rebuild:
rebuild_all_charts(user)
else:
build_all_historical_charts(user)
build_charts_since(user)
self.stdout.write(self.style.SUCCESS(f" OK {user}"))
except Exception as e:
self.stderr.write(f" FAILED {user}: {e}")
self.stdout.write("Chart counts after (diff):")
print_chart_summary(before_counts)
self.stdout.write(self.style.SUCCESS("Done!"))

10
vrobbler/apps/charts/models.py
View File

@ -136,3 +136,13 @@ class ChartRecord(TimeStampedModel):
def get_absolute_url(self):
return reverse("charts:charts-home")
@property
def chart_url(self) -> str:
url = reverse("charts:charts-home")
date_str = str(self.year)
if self.week:
date_str = f"{self.year}-W{self.week}"
elif self.month:
date_str = f"{self.year}-{self.month:02d}"
return f"{url}?date={date_str}"

121
vrobbler/apps/charts/utils.py
View File

@ -29,12 +29,8 @@ def get_time_filter(
tz = pytz.timezone(settings.TIME_ZONE)
if day and month:
start = timezone.make_aware(
datetime.datetime(year, month, day, 0, 0, 0)
)
end = timezone.make_aware(
datetime.datetime(year, month, day, 23, 59, 59)
)
start = timezone.make_aware(datetime.datetime(year, month, day, 0, 0, 0))
end = timezone.make_aware(datetime.datetime(year, month, day, 23, 59, 59))
elif week:
period = datetime.date.fromisocalendar(year, week, 1)
start = timezone.make_aware(
@ -44,9 +40,7 @@ def get_time_filter(
elif month:
last_day = calendar.monthrange(year, month)[1]
start = timezone.make_aware(datetime.datetime(year, month, 1))
end = timezone.make_aware(
datetime.datetime(year, month, last_day, 23, 59, 59)
)
end = timezone.make_aware(datetime.datetime(year, month, last_day, 23, 59, 59))
else:
start = timezone.make_aware(datetime.datetime(year, 1, 1))
end = timezone.make_aware(datetime.datetime(year, 12, 31, 23, 59, 59))
@ -77,18 +71,23 @@ def build_charts(
"podcast_episode",
"board_game",
"trail",
"geo_location",
"food",
"book",
]
period_filter = Q(year=year)
if month:
period_filter &= Q(month=month)
if week:
period_filter &= Q(week=week)
if day:
period_filter &= Q(day=day)
period_filter = (
period_filter & Q(day=day) & Q(week__isnull=True) & Q(month__isnull=True)
)
elif week:
period_filter = (
period_filter & Q(week=week) & Q(day__isnull=True) & Q(month__isnull=True)
)
elif month:
period_filter = (
period_filter & Q(month=month) & Q(week__isnull=True) & Q(day__isnull=True)
)
ChartRecord.objects.filter(period_filter, user=user).delete()
@ -115,8 +114,7 @@ def build_charts(
"annotate": Count("id", distinct=True),
},
"tv_series": {
"filter": Q(video__isnull=False)
& Q(video__tv_series__isnull=False),
"filter": Q(video__isnull=False) & Q(video__tv_series__isnull=False),
"values": "video__tv_series",
"annotate": Count("id", distinct=True),
},
@ -181,12 +179,8 @@ def build_charts(
if not results:
continue
unique_counts = sorted(
set(r["scrobble_count"] for r in results), reverse=True
)
ranks = {
count: rank for rank, count in enumerate(unique_counts, start=1)
}
unique_counts = sorted(set(r["scrobble_count"] for r in results), reverse=True)
ranks = {count: rank for rank, count in enumerate(unique_counts, start=1)}
chart_records = []
total_created = 0
@ -204,16 +198,12 @@ def build_charts(
chart_records.append(ChartRecord(**chart_record_data))
if len(chart_records) >= BATCH_SIZE:
ChartRecord.objects.bulk_create(
chart_records, batch_size=BATCH_SIZE
)
ChartRecord.objects.bulk_create(chart_records, batch_size=BATCH_SIZE)
total_created += len(chart_records)
chart_records = []
if chart_records:
ChartRecord.objects.bulk_create(
chart_records, batch_size=BATCH_SIZE
)
ChartRecord.objects.bulk_create(chart_records, batch_size=BATCH_SIZE)
total_created += len(chart_records)
logger.info(
@ -266,9 +256,7 @@ def build_monthly_charts(
)
def build_yearly_charts(
user, year: int, media_types: Optional[list] = None
) -> None:
def build_yearly_charts(user, year: int, media_types: Optional[list] = None) -> None:
"""Build yearly charts for a specific year."""
build_charts(
user=user,
@ -277,9 +265,7 @@ def build_yearly_charts(
)
def build_all_historical_charts(
user, media_types: Optional[list] = None
) -> None:
def build_all_historical_charts(user, media_types: Optional[list] = None) -> None:
"""Build all historical charts for a user."""
Scrobble = apps.get_model("scrobbles", "Scrobble")
@ -325,12 +311,75 @@ def build_daily_charts(
build_charts(
user=user,
year=year,
month=month,
day=day,
media_types=media_types,
)
def build_charts_since(user, media_types: Optional[list] = None) -> None:
"""Build/update charts starting from the last known ChartRecord date."""
ChartRecord = apps.get_model("charts", "ChartRecord")
Scrobble = apps.get_model("scrobbles", "Scrobble")
latest_daily = (
ChartRecord.objects.filter(user=user, day__isnull=False)
.order_by("-year", "-month", "-day")
.first()
)
if latest_daily:
start_date = datetime.date(
latest_daily.year, latest_daily.month, latest_daily.day
)
logger.info(
f"Building charts since {start_date} for {user} (last known record)"
)
else:
first_scrobble = (
Scrobble.objects.filter(user=user, played_to_completion=True)
.order_by("timestamp")
.first()
)
if not first_scrobble:
logger.info(f"No scrobbles found for {user}")
return
start_date = first_scrobble.timestamp.date()
logger.info(
f"No existing charts found for {user}, building from "
f"first scrobble: {start_date}"
)
start_date -= datetime.timedelta(days=1)
last_week = start_date.isocalendar()[1]
last_month = start_date.month
last_year = start_date.year
now = timezone.now()
current = start_date + datetime.timedelta(days=1)
while current <= now.date():
year = current.year
month = current.month
day = current.day
week = current.isocalendar()[1]
if last_month != month:
build_monthly_charts(user, year, month, media_types)
last_month = month
build_daily_charts(user, year, month, day, media_types)
if last_week != week:
build_weekly_charts(user, year, week, media_types)
last_week = week
if last_year != year:
build_yearly_charts(user, year, media_types)
last_year = year
current += datetime.timedelta(days=1)
def rebuild_all_charts(user, media_types: Optional[list] = None) -> None:
"""Delete all existing chart records for a user and rebuild them."""
ChartRecord = apps.get_model("charts", "ChartRecord")

8
vrobbler/apps/foods/views.py
View File

@ -1,11 +1,15 @@
from foods.models import Food
from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView
from scrobbles.views import (
ScrobbleableListView,
ScrobbleableDetailView,
ChartContextMixin,
)
class FoodListView(ScrobbleableListView):
model = Food
class FoodDetailView(ScrobbleableDetailView):
class FoodDetailView(ScrobbleableDetailView, ChartContextMixin):
model = Food

3
vrobbler/apps/locations/views.py
View File

@ -2,6 +2,7 @@ from django.db.models import Count
from django.views import generic
from locations.models import GeoLocation
from scrobbles.stats import get_scrobble_count_qs
from scrobbles.views import ChartContextMixin
class GeoLocationListView(generic.ListView):
@ -22,6 +23,6 @@ class GeoLocationListView(generic.ListView):
return context_data
class GeoLocationDetailView(generic.DetailView):
class GeoLocationDetailView(ChartContextMixin, generic.DetailView):
model = GeoLocation
slug_field = "uuid"

25
vrobbler/apps/music/views.py
View File

@ -47,19 +47,28 @@ class ArtistDetailView(generic.DetailView):
def get_context_data(self, **kwargs):
context_data = super().get_context_data(**kwargs)
artist = context_data["object"]
rank = 1
tracks_ranked = []
scrobbles = artist.tracks.first().scrobble_count
for track in artist.tracks:
if scrobbles > track.scrobble_count:
rank += 1
tracks_ranked.append((rank, track))
scrobbles = track.scrobble_count
ranked_tracks = sorted(
artist.tracks.all(), key=lambda t: t.scrobble_count, reverse=True
)[:10]
tracks_ranked = [
(rank, track) for rank, track in enumerate(ranked_tracks, start=1)
]
context_data["tracks_ranked"] = tracks_ranked
context_data["charts"] = ChartRecord.objects.filter(
artist=self.object, rank__in=[1, 2, 3]
)
from scrobbles.models import Scrobble
context_data["recent_scrobbles"] = (
Scrobble.objects.filter(track__artist=artist)
.select_related("track", "track__album")
.order_by("-timestamp")[:100]
)
return context_data

8
vrobbler/apps/podcasts/views.py
View File

@ -4,7 +4,11 @@ from django.views import generic
from podcasts.models import Podcast, PodcastEpisode
from scrobbles.models import Scrobble
from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView
from scrobbles.views import (
ScrobbleableListView,
ScrobbleableDetailView,
ChartContextMixin,
)
class PodcastListView(ScrobbleableListView):
@ -61,7 +65,7 @@ class PodcastListView(ScrobbleableListView):
return context_data
class PodcastDetailView(generic.DetailView):
class PodcastDetailView(ChartContextMixin, generic.DetailView):
model = Podcast
slug_field = "uuid"

22
vrobbler/apps/scrobbles/tasks.py
View File

@ -1,7 +1,7 @@
import logging
from celery import shared_task
from charts.utils import build_all_historical_charts, build_yesterdays_charts
from charts.utils import build_charts_since
from django.apps import apps
from django.contrib.auth import get_user_model
@ -53,27 +53,17 @@ def process_koreader_import(import_id):
@shared_task
def create_yesterdays_charts():
"""Build/update charts for all users starting from last known record."""
for user in User.objects.all():
build_yesterdays_charts(user)
@shared_task
def build_missing_charts_for_all_users():
"""Build missing charts for all users."""
for user in User.objects.all():
logger.info(f"Building all historical charts for {user}")
try:
build_all_historical_charts(user)
except Exception as e:
logger.error(f"Failed to build charts for {user}: {e}")
build_charts_since(user)
@shared_task
def build_charts_for_user(user_id):
"""Build all historical charts for a specific user."""
"""Build charts for a specific user starting from last known record."""
user = User.objects.filter(id=user_id).first()
if not user:
logger.error(f"User with id {user_id} not found")
return
logger.info(f"Building all historical charts for {user}")
build_all_historical_charts(user)
logger.info(f"Building charts for {user}")
build_charts_since(user)

36
vrobbler/apps/scrobbles/views.py
View File

@ -105,7 +105,40 @@ class ScrobbleableListView(ListView):
return queryset
class ScrobbleableDetailView(DetailView):
class ChartContextMixin:
def get_context_data(self, **kwargs):
context = super().get_context_data(**kwargs)
from charts.models import ChartRecord
obj = context.get("object")
if not obj:
return context
media_type_map = {
"music.Artist": "artist",
"music.Album": "album",
"music.Track": "track",
"videos.video": "video",
"videos.series": "tv_series",
"podcasts.podcast": "podcast",
"boardgames.boardgame": "board_game",
"trails.trail": "trail",
"foods.food": "food",
"books.book": "book",
}
model_label = f"{obj._meta.app_label}.{obj._meta.model_name}"
media_type = media_type_map.get(model_label)
if media_type:
context["charts"] = ChartRecord.objects.filter(
**{media_type: obj}, rank__in=[1, 2, 3]
)
return context
class ScrobbleableDetailView(ChartContextMixin, DetailView):
model = None
slug_field = "uuid"
@ -734,7 +767,6 @@ class ScrobbleStatusView(LoginRequiredMixin, TemplateView):
def get_context_data(self, **kwargs):
data = super().get_context_data(**kwargs)
user_scrobble_qs = Scrobble.objects.filter().order_by("-timestamp")
progress_plays = user_scrobble_qs.filter(in_progress=True, is_paused=False)
data["listening"] = progress_plays.filter(
Q(track__isnull=False) | Q(podcast_episode__isnull=False)

8
vrobbler/apps/videos/views.py
View File

@ -4,7 +4,11 @@ from django.utils import timezone
from django.views import generic
from scrobbles.models import Scrobble
from videos.models import Channel, Series, Video
from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView
from scrobbles.views import (
ScrobbleableListView,
ScrobbleableDetailView,
ChartContextMixin,
)
class MovieListView(LoginRequiredMixin, generic.ListView):
@ -19,7 +23,7 @@ class SeriesListView(LoginRequiredMixin, generic.ListView):
model = Series
class SeriesDetailView(LoginRequiredMixin, generic.DetailView):
class SeriesDetailView(LoginRequiredMixin, ChartContextMixin, generic.DetailView):
model = Series
slug_field = "uuid"

7
vrobbler/templates/boardgames/boardgame_detail.html
View File

@ -47,6 +47,13 @@
<a href="{{object.start_url}}">Play again</a>
</p>
</div>
{% if charts %}
<div class="row">
<div class="col-md">
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
</div>
</div>
{% endif %}
<div class="row">
<div class="col-md">
<h3>Last scrobbles</h3>

7
vrobbler/templates/books/book_detail.html
View File

@ -41,6 +41,13 @@
{% endif %}
{% endfor %}
</div>
{% if charts %}
<div class="row">
<div class="col-md">
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
</div>
</div>
{% endif %}
<div class="row">
<div class="col-md">
<h3>Last scrobbles</h3>

8
vrobbler/templates/foods/food_detail.html
View File

@ -9,6 +9,14 @@
<p>{{object.description}}</p>
</div>
</div>
{{charts}}
{% if charts %}
<div class="row">
<div class="col-md">
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
</div>
</div>
{% endif %}
<div class="row">
<div class="col-md">
<h3>Last scrobbles</h3>

2
vrobbler/templates/music/album_detail.html
View File

@ -33,7 +33,7 @@
<div class="row">
<p>{{object.scrobbles.count}} scrobbles</p>
{% if charts %}
<p>{% for chart in charts %}<em><a href="{{chart.link}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
{% endif %}
<div class="col-md">
<h3>Top tracks</h3>

4
vrobbler/templates/music/artist_detail.html
View File

@ -34,7 +34,7 @@
<div class="row">
<p>{{artist.scrobbles.count}} scrobbles</p>
{% if charts %}
<p>{% for chart in charts %}<em><a href="{{chart.link}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
{% endif %}
<div class="col-md">
<h3>Top tracks</h3>
@ -81,7 +81,7 @@
</tr>
</thead>
<tbody>
{% for scrobble in object.scrobbles %}
{% for scrobble in recent_scrobbles %}
<tr>
<td><a href={{scrobble.get_absolute_url}}>{{scrobble.local_timestamp}}</a></td>
<td><a href="{{scrobble.track.get_absolute_url}}">{{scrobble.track.title}}</a></td>

2
vrobbler/templates/music/track_detail.html
View File

@ -11,7 +11,7 @@
<div class="row">
<p>{{scrobbles.count}} scrobbles</p>
{% if charts %}
<p>{% for chart in charts %}<em><a href="{{chart.link}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
{% endif %}
<div class="col-md">
<h3>Last scrobbles</h3>

7
vrobbler/templates/podcasts/podcast_detail.html
View File

@ -31,6 +31,13 @@
<div class="deets">
</div>
</div>
{% if charts %}
<div class="row">
<div class="col-md">
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
</div>
</div>
{% endif %}
<div class="row">
<div class="col-md">
<h3>Last scrobbles</h3>

7
vrobbler/templates/trails/trail_detail.html
View File

@ -44,6 +44,13 @@
<a href="{{object.start_url}}">Play again</a>
</p>
</div>
{% if charts %}
<div class="row">
<div class="col-md">
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
</div>
</div>
{% endif %}
<div class="row">
<div class="col-md">
<h3>Last scrobbles</h3>

7
vrobbler/templates/videos/series_detail.html
View File

@ -44,6 +44,13 @@
</p>
</div>
</div>
{% if charts %}
<div class="row">
<div class="col-md">
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
</div>
</div>
{% endif %}
<div class="row">
<div class="col-md">
<h3>Last scrobbles</h3>

7
vrobbler/templates/videos/video_detail.html
View File

@ -83,18 +83,25 @@ dd {
</div>
<div class="row">
<div class="col-md">
{% if charts %}
<p>{% for chart in charts %}<em><a href="{{chart.chart_url}}">{{chart}}</a></em>{% if forloop.last %}{% else %} | {% endif %}{% endfor %}</p>
{% endif %}
<h3>Last scrobbles</h3>
<div class="table-responsive">
<table class="table table-striped table-sm">
<thead>
<tr>
<th scope="col">Date</th>
<th scope="col">With</th>
<th scope="col">Rated</th>
</tr>
</thead>
<tbody>
{% for scrobble in scrobbles.all %}
<tr>
<td><a href={{scrobble.get_absolute_url}}>{{scrobble.local_timestamp}}</a></td>
<td>{% if scrobble.logdata.with_people %}{{scrobble.logdata.with_people|join:", " }}{% else %}Solo{% endif %}</td>
<td>{% if scrobble.logdata.rating %}{{scrobble.logdata.rating }}{% else %}Unrated{% endif %}</td>
</tr>
{% endfor %}
</tbody>