secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Big fix to aggregation

Browse Source
This commit is contained in:
Colin Powell
2023-02-26 22:21:49 -05:00
parent d5bf6440b0
commit 457afdc9ef
15 changed files with 599 additions and 365 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
vrobbler/apps/music/aggregators.py
View File

@ -78,13 +78,13 @@ def week_of_scrobbles(
media_filter = Q(video__video_type=Video.VideoType.TV_EPISODE)
for day in range(6, -1, -1):
start = start - timedelta(days=day)
end = datetime.combine(start, datetime.max.time(), now.tzinfo)
day_of_week = start.strftime('%A')
start_day = start - timedelta(days=day)
end = datetime.combine(start_day, datetime.max.time(), now.tzinfo)
day_of_week = start_day.strftime('%A')
scrobble_day_dict[day_of_week] = base_qs.filter(
media_filter,
timestamp__gte=start,
timestamp__gte=start_day,
timestamp__lte=end,
played_to_completion=True,
).count()
@ -93,7 +93,7 @@ def week_of_scrobbles(
def top_tracks(
user: "User", filter: str = "today", limit: int = 15
user: "User", filter: str = "today", limit: int = 30
) -> List["Track"]:
now = timezone.now()
@ -109,7 +109,9 @@ def top_tracks(
starting_day_of_current_month = now.date().replace(day=1)
starting_day_of_current_year = now.date().replace(month=1, day=1)
time_filter = Q(scrobble__timestamp__gte=start_of_today)
time_filter = Q()
if filter == "today":
time_filter = Q(scrobble__timestamp__gte=start_of_today)
if filter == "week":
time_filter = Q(scrobble__timestamp__gte=starting_day_of_current_week)
if filter == "month":
@ -119,7 +121,13 @@ def top_tracks(
return (
Track.objects.filter(time_filter)
.annotate(num_scrobbles=Count("scrobble", distinct=True))
.annotate(
num_scrobbles=Count(
"scrobble",
filter=Q(scrobble__played_to_completion=True),
distinct=True,
)
)
.order_by("-num_scrobbles")[:limit]
)
@ -143,8 +151,14 @@ def top_artists(
return (
Artist.objects.filter(time_filter)
.annotate(num_scrobbles=Count("track__scrobble", distinct=True))
.order_by("-num_scrobbles")[:limit]
.annotate(
num_scrobbles=Count(
"track__scrobble",
filter=Q(track__scrobble__played_to_completion=True),
distinct=True,
)
)
.order_by("-num_scrobbles")
)

37
vrobbler/apps/music/utils.py
View File

@ -1,6 +1,8 @@
import logging
import re
from musicbrainzngs.caa import musicbrainz
from scrobbles.musicbrainz import (
lookup_album_dict_from_mb,
lookup_artist_from_mb,
@ -21,20 +23,20 @@ def get_or_create_artist(name: str, mbid: str = None) -> Artist:
name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip()
if 'featuring' in name.lower():
name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip()
if '&' in name.lower():
name = re.split("&", name, flags=re.IGNORECASE)[0].strip()
artist_dict = lookup_artist_from_mb(name)
mbid = mbid or artist_dict['id']
logger.debug(f'Looking up artist {name} and mbid: {mbid}')
artist, created = Artist.objects.get_or_create(
name=name, musicbrainz_id=mbid
)
logger.debug(f"Cleaning artist {name} with {artist_dict['name']}")
# Clean up bad names in our DB with MB names
# if artist.name != artist_dict["name"]:
# artist.name = artist_dict["name"]
# artist.save(update_fields=["name"])
artist = Artist.objects.filter(musicbrainz_id=mbid).first()
if not artist:
artist = Artist.objects.create(name=name, musicbrainz_id=mbid)
logger.debug(
f"Created artist {artist.name} ({artist.musicbrainz_id}) "
)
# TODO Enrich artist with MB data
return artist
@ -78,27 +80,22 @@ def get_or_create_album(name: str, artist: Artist, mbid: str = None) -> Album:
def get_or_create_track(
title: str,
mbid: str,
artist: Artist,
album: Album,
mbid: str = None,
run_time=None,
run_time_ticks=None,
) -> Track:
track = None
if mbid:
track = Track.objects.filter(
musicbrainz_id=mbid,
).first()
if not track:
track = Track.objects.filter(
title=title, artist=artist, album=album
).first()
if not mbid:
mbid = lookup_track_from_mb(
title, artist.musicbrainz_id, album.musicbrainz_id
title,
artist.musicbrainz_id,
album.musicbrainz_id,
)['id']
track = Track.objects.filter(musicbrainz_id=mbid).first()
if not track:
track = Track.objects.create(
title=title,

1
vrobbler/apps/scrobbles/admin.py
View File

@ -47,6 +47,7 @@ class ChartRecordAdmin(admin.ModelAdmin):
list_display = (
"user",
"rank",
"count",
"year",
"week",
"month",

19
vrobbler/apps/scrobbles/context_processors.py Normal file
View File

@ -0,0 +1,19 @@
import pytz
from django.utils import timezone
from scrobbles.models import Scrobble
def now_playing(request):
user = request.user
now = timezone.now()
if user.is_authenticated:
if user.profile:
timezone.activate(pytz.timezone(user.profile.timezone))
now = timezone.localtime(timezone.now())
return {
'now_playing_list': Scrobble.objects.filter(
in_progress=True,
is_paused=False,
user=user,
)
}

47
vrobbler/apps/scrobbles/models.py
View File

@ -2,18 +2,19 @@ import calendar
import logging
from uuid import uuid4
from books.models import Book
from django.contrib.auth import get_user_model
from django.db import models
from django.urls import reverse
from django.utils import timezone
from django_extensions.db.models import TimeStampedModel
from music.models import Artist, Track
from books.models import Book
from podcasts.models import Episode
from profiles.utils import now_user_timezone
from scrobbles.lastfm import LastFM
from scrobbles.utils import check_scrobble_for_finish
from sports.models import SportEvent
from videos.models import Series, Video
from vrobbler.apps.scrobbles.stats import build_charts
logger = logging.getLogger(__name__)
@ -35,6 +36,24 @@ class BaseFileImportMixin(TimeStampedModel):
def __str__(self):
return f"Scrobble import {self.id}"
@property
def human_start(self):
start = "Unknown"
if self.processing_started:
start = self.processing_started.strftime('%B %d, %Y at %H:%M')
return start
@property
def import_type(self) -> str:
class_name = self.__class__.__name__
if class_name == 'AudioscrobblerTSVImport':
return "Audioscrobbler"
if class_name == 'KoReaderImport':
return "KoReader"
if self.__class__.__name__ == 'LastFMImport':
return "LastFM"
return "Generic"
def process(self, force=False):
logger.warning("Process not implemented")
@ -99,6 +118,14 @@ class KoReaderImport(BaseFileImportMixin):
class Meta:
verbose_name = "KOReader Import"
def __str__(self):
return f"KoReader import on {self.human_start}"
def get_absolute_url(self):
return reverse(
'scrobbles:koreader-import-detail', kwargs={'slug': self.uuid}
)
def get_path(instance, filename):
extension = filename.split('.')[-1]
uuid = instance.uuid
@ -127,6 +154,14 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
class Meta:
verbose_name = "AudioScrobbler TSV Import"
def __str__(self):
return f"Audioscrobbler import on {self.human_start}"
def get_absolute_url(self):
return reverse(
'scrobbles:tsv-import-detail', kwargs={'slug': self.uuid}
)
def get_path(instance, filename):
extension = filename.split('.')[-1]
uuid = instance.uuid
@ -159,6 +194,14 @@ class LastFmImport(BaseFileImportMixin):
class Meta:
verbose_name = "Last.FM Import"
def __str__(self):
return f"LastFM import on {self.human_start}"
def get_absolute_url(self):
return reverse(
'scrobbles:lastfm-import-detail', kwargs={'slug': self.uuid}
)
def process(self, import_all=False):
"""Import scrobbles found on LastFM"""
if self.processed_finished:

7
vrobbler/apps/scrobbles/scrobblers.py
View File

@ -89,10 +89,6 @@ def mopidy_scrobble_track(
"mopidy_status": data_dict.get("status"),
}
# Jellyfin MB ids suck, so always overwrite with Mopidy if they're offering
track.musicbrainz_id = data_dict.get("musicbrainz_track_id")
track.save()
scrobble = Scrobble.create_or_update(track, user_id, mopidy_data)
return scrobble
@ -130,7 +126,7 @@ def jellyfin_scrobble_track(
)
# Jellyfin has some race conditions with it's webhooks, these hacks fix some of them
if not data_dict.get("PlaybackPositionTicks") or null_position_on_progress:
if null_position_on_progress:
logger.error("No playback position tick from Jellyfin, aborting")
return
@ -152,7 +148,6 @@ def jellyfin_scrobble_track(
)
track = get_or_create_track(
title=data_dict.get("Name"),
mbid=data_dict.get(JELLYFIN_POST_KEYS["TRACK_MB_ID"]),
artist=artist,
album=album,
run_time_ticks=run_time_ticks,

4
vrobbler/apps/scrobbles/stats.py
View File

@ -137,4 +137,6 @@ def build_charts(
if model_str == 'Artist':
chart_record['artist'] = result
chart_records.append(ChartRecord(**chart_record))
ChartRecord.objects.bulk_create(chart_records, ignore_conflicts=True)
ChartRecord.objects.bulk_create(
chart_records, ignore_conflicts=True, batch_size=500
)

20
vrobbler/apps/scrobbles/urls.py
View File

@ -42,6 +42,26 @@ urlpatterns = [
name='mopidy-webhook',
),
path('export/', views.export, name='export'),
path(
'imports/',
views.ScrobbleImportListView.as_view(),
name='import-detail',
),
path(
'imports/tsv/<slug:slug>/',
views.ScrobbleTSVImportDetailView.as_view(),
name='tsv-import-detail',
),
path(
'imports/lastfm/<slug:slug>/',
views.ScrobbleLastFMImportDetailView.as_view(),
name='lastfm-import-detail',
),
path(
'imports/koreader/<slug:slug>/',
views.ScrobbleKoReaderImportDetailView.as_view(),
name='koreader-import-detail',
),
path(
'charts/',
views.ChartRecordView.as_view(),

201
vrobbler/apps/scrobbles/views.py
View File

@ -1,7 +1,8 @@
import calendar
import json
import logging
from datetime import datetime
from datetime import datetime, timedelta
from django.db.models.query import QuerySet
import pytz
from django.conf import settings
@ -13,7 +14,7 @@ from django.http import FileResponse, HttpResponseRedirect, JsonResponse
from django.urls import reverse, reverse_lazy
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import FormView, TemplateView
from django.views.generic import DetailView, FormView, TemplateView
from django.views.generic.edit import CreateView
from django.views.generic.list import ListView
from music.aggregators import (
@ -70,17 +71,7 @@ class RecentScrobbleList(ListView):
def get_context_data(self, **kwargs):
data = super().get_context_data(**kwargs)
user = self.request.user
now = timezone.now()
if user.is_authenticated:
if user.profile:
timezone.activate(pytz.timezone(user.profile.timezone))
now = timezone.localtime(timezone.now())
data['now_playing_list'] = Scrobble.objects.filter(
in_progress=True,
is_paused=False,
timestamp__lte=now,
user=user,
)
completed_for_user = Scrobble.objects.filter(
played_to_completion=True, user=user
@ -97,19 +88,16 @@ class RecentScrobbleList(ListView):
sport_event__isnull=False
).order_by('-timestamp')[:15]
# data['top_daily_tracks'] = top_tracks()
data['top_weekly_tracks'] = top_tracks(user, filter='week')
data['top_monthly_tracks'] = top_tracks(user, filter='month')
# data['top_daily_artists'] = top_artists()
data['top_weekly_artists'] = top_artists(user, filter='week')
data['top_monthly_artists'] = top_artists(user, filter='month')
data["weekly_data"] = week_of_scrobbles(user=user)
data['counts'] = scrobble_counts(user)
data['imdb_form'] = ScrobbleForm
data['export_form'] = ExportScrobbleForm
data['active_imports'] = AudioScrobblerTSVImport.objects.filter(
processing_started__isnull=False,
processed_finished__isnull=True,
user=self.request.user,
)
return data
def get_queryset(self):
@ -118,6 +106,57 @@ class RecentScrobbleList(ListView):
).order_by('-timestamp')[:15]
class ScrobbleImportListView(TemplateView):
template_name = "scrobbles/import_list.html"
def get_context_data(self, **kwargs):
context_data = super().get_context_data(**kwargs)
context_data['object_list'] = []
context_data["tsv_imports"] = AudioScrobblerTSVImport.objects.filter(
user=self.request.user,
).order_by('-processing_started')
context_data["koreader_imports"] = KoReaderImport.objects.filter(
user=self.request.user,
).order_by('-processing_started')
context_data["lastfm_imports"] = LastFmImport.objects.filter(
user=self.request.user,
).order_by('-processing_started')
return context_data
class BaseScrobbleImportDetailView(DetailView):
slug_field = 'uuid'
template_name = "scrobbles/import_detail.html"
def get_queryset(self):
return super().get_queryset().filter(user=self.request.user)
def get_context_data(self, **kwargs):
context_data = super().get_context_data(**kwargs)
title = "Generic Scrobble Import"
if self.model == KoReaderImport:
title = "KoReader Import"
if self.model == AudioScrobblerTSVImport:
title = "Audioscrobbler TSV Import"
if self.model == LastFmImport:
title = "LastFM Import"
context_data['title'] = title
return context_data
class ScrobbleKoReaderImportDetailView(BaseScrobbleImportDetailView):
model = KoReaderImport
class ScrobbleTSVImportDetailView(BaseScrobbleImportDetailView):
model = AudioScrobblerTSVImport
class ScrobbleLastFMImportDetailView(BaseScrobbleImportDetailView):
model = LastFmImport
class ManualScrobbleView(FormView):
form_class = ScrobbleForm
template_name = 'scrobbles/manual_form.html'
@ -366,44 +405,116 @@ def export(request):
class ChartRecordView(TemplateView):
template_name = 'scrobbles/chart_index.html'
def get_context_data(self, **kwargs):
context_data = super().get_context_data(**kwargs)
date = self.request.GET.get('date')
media_type = self.request.GET.get('media')
params = {}
if not media_type:
media_type = 'Track'
context_data['media_type'] = media_type
media_filter = Q(track__isnull=False)
if media_type == 'Video':
media_filter = Q(video__isnull=False)
@staticmethod
def get_media_filter(media_type: str = "Track"):
media_filter = Q()
if media_type == 'Track':
media_filter = Q(track__isnull=False)
if media_type == 'Artist':
media_filter = Q(artist__isnull=False)
if media_type == 'Series':
media_filter = Q(series__isnull=False)
if media_type == 'Video':
media_filter = Q(video__isnull=False)
return media_filter
year = timezone.now().year
def get_chart_records(self, media_type: str = "Track", **kwargs):
media_filter = self.get_media_filter(media_type)
charts = ChartRecord.objects.filter(
media_filter, user=self.request.user, **kwargs
).order_by("rank")
if charts.count() == 0:
ChartRecord.build(
user=self.request.user, model_str=media_type, **kwargs
)
charts = ChartRecord.objects.filter(
media_filter, user=self.request.user, **kwargs
).order_by("rank")
return charts
def get_chart(
self, period: str = "all_time", limit=15, media: str = "Track"
) -> QuerySet:
chart = QuerySet()
now = timezone.now()
if period == "all_time":
chart = self.get_chart_records(media_type=media)
if period == "today":
chart = self.get_chart_records(
media_type=media,
day=now.day,
month=now.month,
year=now.year,
)
if period == "week":
chart = self.get_chart_records(
media_type=media,
year=now.year,
week=now.isocalendar()[1],
)
if period == "month":
chart = self.get_chart_records(
media_type=media,
year=now.year,
month=now.month,
)
if period == "year":
chart = self.get_chart_records(
media_type=media,
year=now.year,
)
return chart[:limit]
def get_context_data(self, **kwargs):
context_data = super().get_context_data(**kwargs)
date = self.request.GET.get('date')
media_type = self.request.GET.get('media')
user = self.request.user
params = {}
context_data['artist_charts'] = {}
if not date:
context_data['artist_charts'] = {
"today": top_artists(user, filter="today")[:30],
"week": top_artists(user, filter="week")[:30],
"month": top_artists(user, filter="month")[:30],
"all": top_artists(user),
}
context_data['track_charts'] = {
"today": top_tracks(user, filter="today")[:30],
"week": top_tracks(user, filter="week")[:30],
"month": top_tracks(user, filter="month")[:30],
"all": top_tracks(user),
}
return context_data
now = timezone.now()
year = now.year
params = {'year': year}
name = f"Chart for {year}"
if not date:
date = timezone.now().strftime("%Y-%m-%d")
date_params = date.split('-')
year = int(date_params[0])
in_progress = False
if len(date_params) == 2:
if 'W' in date_params[1]:
week = int(date_params[1].strip('W"'))
params['week'] = week
r = datetime.strptime(date + '-1', "%Y-W%W-%w").strftime(
'Week of %B %d, %Y'
start = datetime.strptime(date + "-1", "%Y-W%W-%w").replace(
tzinfo=pytz.utc
)
name = f"Chart for {r}"
end = start + timedelta(days=6)
in_progress = start <= now <= end
as_str = start.strftime('Week of %B %d, %Y')
name = f"Chart for {as_str}"
else:
month = int(date_params[1])
params['month'] = month
month_str = calendar.month_name[month]
name = f"Chart for {month_str} {year}"
in_progress = now.month == month and now.year == year
if len(date_params) == 3:
month = int(date_params[1])
day = int(date_params[2])
@ -411,7 +522,11 @@ class ChartRecordView(TemplateView):
params['day'] = day
month_str = calendar.month_name[month]
name = f"Chart for {month_str} {day}, {year}"
in_progress = (
now.month == month and now.year == year and now.day == day
)
media_filter = self.get_media_filter(media_type)
charts = ChartRecord.objects.filter(
media_filter, user=self.request.user, **params
).order_by("rank")
@ -424,6 +539,10 @@ class ChartRecordView(TemplateView):
media_filter, user=self.request.user, **params
).order_by("rank")
context_data['object_list'] = charts
if in_progress:
# TODO recalculate
...
context_data['name'] = name
context_data['in_progress'] = in_progress
return context_data