Compare commits

...

16 Commits
0.7.5 ... 0.8.2

Author SHA1 Message Date
541073aae3 Bump version to 0.8.2 2023-02-06 19:32:15 -05:00
b63ec6b15f Fix bug in export when artist does not exist 2023-02-06 19:31:25 -05:00
117157e3ae Fix audioscrobbler import bug
Issue was not having a user so we couldn't set a timezone. All fixed now
2023-02-06 19:30:58 -05:00
0c10e78d5e Fix bug in unified scrobbler for podcasts 2023-02-06 17:54:48 -05:00
6b7359707b Bump version to 0.8.1 2023-02-06 01:12:33 -05:00
e0295cbd56 Fix jellyfin edge case scrobbling mess
Finally get to resolve scrobbling music from Jellyfin. This may lead to
other issues, in fact now videos seem to sometimes create duplicate
scrobbles. But music can be scrobbled now from Jellyfin web or Finamp
successfully.
2023-02-06 00:22:10 -05:00
5271cfaea4 Create sport if it doesn't exist yet 2023-02-06 00:19:47 -05:00
0370b64351 Fix exporting only tracks by default 2023-02-06 00:19:07 -05:00
9ec31ba0f5 Remove noisy debug logging 2023-02-05 01:59:24 -05:00
a9de298057 Put import and export behind auth 2023-02-05 01:58:19 -05:00
9d303b1b94 Add exporting and importing scrobbles 2023-02-04 17:08:01 -05:00
4c434aeb7c Bump version to 0.8.0 2023-02-03 19:00:32 -05:00
64d9cac09c Update importing to include some logging 2023-02-03 18:59:48 -05:00
c21d6a96fe Fix unused imports in imdb module 2023-02-03 16:53:03 -05:00
e392477dc7 Add import of Audioscrobbler files
Here we add a model for holding Audioscrobbler imports and some code to
process the tab-separated files we get from Rockbox.
2023-02-03 16:45:31 -05:00
12087460f6 Irritating that poetry can't handle prod deps well 2023-01-31 10:44:39 -05:00
29 changed files with 810 additions and 203 deletions

View File

@ -1,6 +1,6 @@
[tool.poetry]
name = "vrobbler"
version = "0.7.5"
version = "0.8.2"
description = ""
authors = ["Colin Powell <colin@unbl.ink>"]
@ -16,7 +16,7 @@ djangorestframework = "^3.13.1"
Markdown = "^3.3.6"
django-filter = "^21.1"
Pillow = "^9.0.1"
psycopg2 = {version = "^2.9.3", extras = ["production"]}
psycopg2 = "^2.9.3"
dj-database-url = "^0.5.0"
django-mathfilters = "^1.0.0"
django-allauth = "^0.50.0"

View File

@ -9,7 +9,7 @@ When we get artwork from Musicbrianz, and it's not found, we should check for
release groups as well. This will stop issues with missing artwork because of
obscure MB release matches.
* DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion perecnt :bug:
* DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion percent :bug:
CLOSED: [2023-01-30 Mon 18:31]
:LOGBOOK:
CLOCK: [2023-01-30 Mon 18:00]--[2023-01-30 Mon 18:31] => 0:31
@ -26,6 +26,33 @@ as complete for the following conditions:
But if we keep listening beyond 90, we should basically ignore updates (or just
update the existing scrobble)
* DONE [#A] Add support for Audioscrobbler tab-separated file uploads :improvement:
CLOSED: [2023-02-03 Fri 16:52]
An example of the format:
#+begin_src csv
,
#AUDIOSCROBBLER/1.1
#TZ/UNKNOWN
#CLIENT/Rockbox sansaclipplus $Revision$
75 Dollar Bill I Was Real I Was Real 4 1015 S 1740494944 64ff5f53-d187-4512-827e-7606c69e66ff
75 Dollar Bill I Was Real I Was Real 4 1015 S 1740494990 64ff5f53-d187-4512-827e-7606c69e66ff
311 311 Down 1 173 S 1740495003 00476c23-fd9e-464b-9b27-a62d69f3d4f4
311 311 Down 1 173 L 1740495049 00476c23-fd9e-464b-9b27-a62d69f3d4f4
311 311 Down 1 173 L 1740495113 00476c23-fd9e-464b-9b27-a62d69f3d4f4
311 311 Random 2 187 S 1740495190 530c09f3-46fe-4d90-b11f-7b63bcb4b373
311 311 Random 2 187 L 1740495194 530c09f3-46fe-4d90-b11f-7b63bcb4b373
311 311 Jackolanterns Weather 3 204 L 1740495382 cc3b2dec-5d99-47ea-8930-20bf258be4ea
311 311 All Mixed Up 4 182 L 1740495586 980a78b5-5bdd-4f50-9e3a-e13261e2817b
311 311 Hive 5 179 L 1740495768 18f6dc98-d3a2-4f81-b967-97359d14c68c
311 311 Guns (Are for Pussies) 6 137 L 1740495948 5e97ed9f-c8cc-4282-9cbe-f8e17aee5128
311 311 Misdirected Hostility 7 179 S 1740496085 61ff2c1a-fc9c-44c3-8da1-5e50a44245af
,
#+end_src
* TODO [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
Given a UUID from musicbrainz, we should be able to scrobble an album or
individual track.
* TODO [#A] Add django-storage to store files on S3 :improvement:
* TODO [#B] Adjust cancel/finish task to use javascript to submit :improvement:
@ -349,3 +376,5 @@ has to re-populate when the server restarts.
An example:
https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
* TODO When updating musicbrainz IDs, clear and run fetch artwrok :improvement:

View File

@ -183,16 +183,7 @@ class Track(ScrobblableMixin):
return
artist, artist_created = Artist.objects.get_or_create(**artist_dict)
if artist_created:
logger.debug(f"Created new artist {artist}")
else:
logger.debug(f"Found album {artist}")
album, album_created = Album.objects.get_or_create(**album_dict)
if album_created:
logger.debug(f"Created new album {album}")
else:
logger.debug(f"Found album {album}")
album.fix_metadata()
if not album.cover_image:
@ -202,9 +193,5 @@ class Track(ScrobblableMixin):
track_dict['artist_id'] = artist.id
track, created = cls.objects.get_or_create(**track_dict)
if created:
logger.debug(f"Created new track: {track}")
else:
logger.debug(f"Found track {track}")
return track

View File

@ -1,3 +1,5 @@
import pytz
from django.contrib.auth import get_user_model
from django.db import models
from django_extensions.db.models import TimeStampedModel
@ -16,3 +18,7 @@ class UserProfile(TimeStampedModel):
def __str__(self):
return f"User profile for {self.user}"
@property
def tzinfo(self):
return pytz.timezone(self.timezone)

View File

@ -1,6 +1,6 @@
from django.contrib import admin
from scrobbles.models import Scrobble
from scrobbles.models import AudioScrobblerTSVImport, Scrobble
class ScrobbleInline(admin.TabularInline):
@ -10,6 +10,13 @@ class ScrobbleInline(admin.TabularInline):
exclude = ('source_id', 'scrobble_log')
@admin.register(AudioScrobblerTSVImport)
class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
date_hierarchy = "created"
list_display = ("uuid", "created", "process_count", "tsv_file")
ordering = ("-created",)
@admin.register(Scrobble)
class ScrobbleAdmin(admin.ModelAdmin):
date_hierarchy = "timestamp"

View File

@ -0,0 +1,67 @@
import csv
import tempfile
from scrobbles.models import Scrobble
from django.db.models import Q
def export_scrobbles(start_date=None, end_date=None, format="AS"):
start_query = Q()
end_query = Q()
if start_date:
start_query = Q(timestamp__gte=start_date)
if start_date:
end_query = Q(timestamp__lte=end_date)
scrobble_qs = Scrobble.objects.filter(
start_query, end_query, track__isnull=False
)
headers = []
extension = 'tsv'
delimiter = '\t'
if format == "as":
headers = [
['#AUDIOSCROBBLER/1.1'],
['#TZ/UTC'],
['#CLIENT/Vrobbler 1.0.0'],
]
if format == "csv":
delimiter = ','
extension = 'csv'
headers = [
[
"artists",
"album",
"title",
"track_number",
"run_time",
"rating",
"timestamp",
"musicbrainz_id",
]
]
with tempfile.NamedTemporaryFile(mode='w', delete=False) as outfile:
writer = csv.writer(outfile, delimiter=delimiter)
for row in headers:
writer.writerow(row)
for scrobble in scrobble_qs:
track = scrobble.track
track_number = 0 # TODO Add track number
track_rating = "S" # TODO implement ratings?
track_artist = track.artist or track.album.primary_artist
row = [
track_artist,
track.album.name,
track.title,
track_number,
track.run_time,
track_rating,
scrobble.timestamp.strftime('%s'),
track.musicbrainz_id,
]
writer.writerow(row)
return outfile.name, extension

View File

@ -1,6 +1,17 @@
from django import forms
class ExportScrobbleForm(forms.Form):
"""Provide options for downloading scrobbles"""
EXPORT_TYPES = (
('as', 'Audioscrobbler'),
('csv', 'CSV'),
('html', 'HTML'),
)
export_type = forms.ChoiceField(choices=EXPORT_TYPES)
class ScrobbleForm(forms.Form):
item_id = forms.CharField(
label="",

View File

@ -1,9 +1,7 @@
import logging
from typing import Optional
from django.utils import timezone
from imdb import Cinemagoer
from videos.models import Video
imdb_client = Cinemagoer()

View File

@ -0,0 +1,52 @@
# Generated by Django 4.1.5 on 2023-02-03 19:50
from django.db import migrations, models
import django_extensions.db.fields
class Migration(migrations.Migration):
dependencies = [
('scrobbles', '0011_chartrecord_user'),
]
operations = [
migrations.CreateModel(
name='AudioScrobblerTSVImport',
fields=[
(
'id',
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name='ID',
),
),
(
'created',
django_extensions.db.fields.CreationDateTimeField(
auto_now_add=True, verbose_name='created'
),
),
(
'modified',
django_extensions.db.fields.ModificationDateTimeField(
auto_now=True, verbose_name='modified'
),
),
(
'tsv_file',
models.FileField(
blank=True,
null=True,
upload_to='audioscrobbler-uploads/%Y/%m-%d/',
),
),
],
options={
'get_latest_by': 'modified',
'abstract': False,
},
),
]

View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.5 on 2023-02-03 20:20
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scrobbles', '0012_audioscrobblertsvimport'),
]
operations = [
migrations.AddField(
model_name='audioscrobblertsvimport',
name='processed_on',
field=models.DateTimeField(blank=True, null=True),
),
]

View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.5 on 2023-02-03 22:53
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scrobbles', '0013_audioscrobblertsvimport_processed_on'),
]
operations = [
migrations.AddField(
model_name='audioscrobblertsvimport',
name='process_log',
field=models.TextField(blank=True, null=True),
),
]

View File

@ -0,0 +1,29 @@
# Generated by Django 4.1.5 on 2023-02-03 23:36
from django.db import migrations, models
import scrobbles.models
import uuid
class Migration(migrations.Migration):
dependencies = [
('scrobbles', '0014_audioscrobblertsvimport_process_log'),
]
operations = [
migrations.AddField(
model_name='audioscrobblertsvimport',
name='uuid',
field=models.UUIDField(default=uuid.uuid4, editable=False),
),
migrations.AlterField(
model_name='audioscrobblertsvimport',
name='tsv_file',
field=models.FileField(
blank=True,
null=True,
upload_to=scrobbles.models.AudioScrobblerTSVImport.get_path,
),
),
]

View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.5 on 2023-02-03 23:44
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('scrobbles', '0015_audioscrobblertsvimport_uuid_and_more'),
]
operations = [
migrations.AddField(
model_name='audioscrobblertsvimport',
name='process_count',
field=models.IntegerField(blank=True, null=True),
),
]

View File

@ -0,0 +1,26 @@
# Generated by Django 4.1.5 on 2023-02-07 00:07
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('scrobbles', '0016_audioscrobblertsvimport_process_count'),
]
operations = [
migrations.AddField(
model_name='audioscrobblertsvimport',
name='user',
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.DO_NOTHING,
to=settings.AUTH_USER_MODEL,
),
),
]

View File

@ -7,6 +7,8 @@ BNULL = {"blank": True, "null": True}
class ScrobblableMixin(TimeStampedModel):
SECONDS_TO_STALE = 1600
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
title = models.CharField(max_length=255, **BNULL)
run_time = models.CharField(max_length=8, **BNULL)

View File

@ -1,6 +1,5 @@
import calendar
import logging
from datetime import timedelta
from uuid import uuid4
from django.contrib.auth import get_user_model
@ -19,6 +18,62 @@ User = get_user_model()
BNULL = {"blank": True, "null": True}
class AudioScrobblerTSVImport(TimeStampedModel):
def get_path(instance, filename):
extension = filename.split('.')[-1]
uuid = instance.uuid
return f'audioscrobbler-uploads/{uuid}.{extension}'
user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
uuid = models.UUIDField(editable=False, default=uuid4)
tsv_file = models.FileField(upload_to=get_path, **BNULL)
processed_on = models.DateTimeField(**BNULL)
process_log = models.TextField(**BNULL)
process_count = models.IntegerField(**BNULL)
def __str__(self):
if self.tsv_file:
return f"Audioscrobbler TSV upload: {self.tsv_file.path}"
return f"Audioscrobbler TSV upload {self.id}"
def save(self, **kwargs):
"""On save, attempt to import the TSV file"""
super().save(**kwargs)
self.process()
return
def process(self, force=False):
from scrobbles.tsv import process_audioscrobbler_tsv_file
if self.processed_on and not force:
logger.info(f"{self} already processed on {self.processed_on}")
return
tz = None
if self.user:
tz = self.user.profile.tzinfo
scrobbles = process_audioscrobbler_tsv_file(self.tsv_file.path, tz=tz)
if scrobbles:
self.process_log = f"Created {len(scrobbles)} scrobbles"
for scrobble in scrobbles:
scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}"
self.process_log += f"\n{scrobble_str}"
self.process_count = len(scrobbles)
else:
self.process_log = f"Created no new scrobbles"
self.process_count = 0
self.processed_on = timezone.now()
self.save(
update_fields=['processed_on', 'process_count', 'process_log']
)
def undo(self, dryrun=True):
from scrobbles.tsv import undo_audioscrobbler_tsv_import
undo_audioscrobbler_tsv_import(self.process_log, dryrun)
class ChartRecord(TimeStampedModel):
"""Sort of like a materialized view for what we could dynamically generate,
but would kill the DB as it gets larger. Collects time-based records
@ -96,6 +151,8 @@ class ChartRecord(TimeStampedModel):
class Scrobble(TimeStampedModel):
"""A scrobble tracks played media items by a user."""
uuid = models.UUIDField(editable=False, **BNULL)
video = models.ForeignKey(Video, on_delete=models.DO_NOTHING, **BNULL)
track = models.ForeignKey(Track, on_delete=models.DO_NOTHING, **BNULL)
@ -134,24 +191,44 @@ class Scrobble(TimeStampedModel):
return 'in-progress'
return 'zombie'
@property
def is_stale(self) -> bool:
"""Mark scrobble as stale if it's been more than an hour since it was updated"""
is_stale = False
now = timezone.now()
seconds_since_last_update = (now - self.modified).seconds
if seconds_since_last_update >= self.media_obj.SECONDS_TO_STALE:
is_stale = True
return is_stale
@property
def percent_played(self) -> int:
if not self.media_obj.run_time_ticks:
logger.warning(
f"{self} has no run_time_ticks value, cannot show percent played"
)
return 100
if not self.playback_position_ticks and self.played_to_completion:
return 100
playback_ticks = self.playback_position_ticks
if not playback_ticks:
playback_ticks = (timezone.now() - self.timestamp).seconds * 1000
if self.played_to_completion:
return 100
percent = int((playback_ticks / self.media_obj.run_time_ticks) * 100)
if percent > 100:
percent = 100
return percent
@property
def can_be_updated(self) -> bool:
updatable = True
if self.percent_played > 100:
logger.info(f"No - 100% played - {self.id} - {self.source}")
updatable = False
if self.is_stale:
logger.info(f"No - stale - {self.id} - {self.source}")
updatable = False
return updatable
@property
def media_obj(self):
media_obj = None
@ -170,158 +247,71 @@ class Scrobble(TimeStampedModel):
return f"Scrobble of {self.media_obj} ({timestamp})"
@classmethod
def create_or_update_for_video(
cls, video: "Video", user_id: int, scrobble_data: dict
def create_or_update(
cls, media, user_id: int, scrobble_data: dict
) -> "Scrobble":
scrobble_data['video_id'] = video.id
if media.__class__.__name__ == 'Track':
media_query = models.Q(track=media)
scrobble_data['track_id'] = media.id
if media.__class__.__name__ == 'Video':
media_query = models.Q(video=media)
scrobble_data['video_id'] = media.id
if media.__class__.__name__ == 'Episode':
media_query = models.Q(podcast_episode=media)
scrobble_data['podcast_episode_id'] = media.id
if media.__class__.__name__ == 'SportEvent':
media_query = models.Q(sport_event=media)
scrobble_data['sport_event_id'] = media.id
scrobble = (
cls.objects.filter(
video=video,
media_query,
user_id=user_id,
)
.order_by('-modified')
.first()
)
if scrobble and scrobble.percent_played <= 100:
if scrobble and scrobble.can_be_updated:
logger.info(
f"Found existing scrobble for video {video}, updating",
{"scrobble_data": scrobble_data},
f"Updating {scrobble.id}",
{"scrobble_data": scrobble_data, "media": media},
)
return cls.update(scrobble, scrobble_data)
return scrobble.update(scrobble_data)
logger.debug(
f"No existing scrobble for video {video}, creating",
{"scrobble_data": scrobble_data},
)
# If creating a new scrobble, we don't need status
scrobble_data.pop('jellyfin_status')
return cls.create(scrobble_data)
@classmethod
def create_or_update_for_track(
cls, track: "Track", user_id: int, scrobble_data: dict
) -> "Scrobble":
"""Look up any existing scrobbles for a track and compare
the appropriate backoff time for music tracks to the setting
so we can avoid duplicating scrobbles."""
scrobble_data['track_id'] = track.id
scrobble = (
cls.objects.filter(
track=track,
user_id=user_id,
played_to_completion=False,
)
.order_by('-modified')
.first()
)
if scrobble:
logger.debug(
f"Found existing scrobble for track {track}, updating",
{"scrobble_data": scrobble_data},
)
return cls.update(scrobble, scrobble_data)
if 'jellyfin_status' in scrobble_data.keys():
last_scrobble = Scrobble.objects.last()
if (
scrobble_data['timestamp'] - last_scrobble.timestamp
).seconds <= 1:
logger.warning('Jellyfin spammed us with duplicate updates')
return last_scrobble
logger.debug(
f"No existing scrobble for track {track}, creating",
{"scrobble_data": scrobble_data},
source = scrobble_data['source']
logger.info(
f"Creating for {media.id} - {source}",
{"scrobble_data": scrobble_data, "media": media},
)
# If creating a new scrobble, we don't need status
scrobble_data.pop('mopidy_status', None)
scrobble_data.pop('jellyfin_status', None)
return cls.create(scrobble_data)
@classmethod
def create_or_update_for_podcast_episode(
cls, episode: "Episode", user_id: int, scrobble_data: dict
) -> "Scrobble":
scrobble_data['podcast_episode_id'] = episode.id
scrobble = (
cls.objects.filter(
podcast_episode=episode,
user_id=user_id,
played_to_completion=False,
)
.order_by('-modified')
.first()
)
if scrobble:
logger.debug(
f"Found existing scrobble for podcast {episode}, updating",
{"scrobble_data": scrobble_data},
)
return cls.update(scrobble, scrobble_data)
logger.debug(
f"No existing scrobble for podcast epsiode {episode}, creating",
{"scrobble_data": scrobble_data},
)
# If creating a new scrobble, we don't need status
scrobble_data.pop('mopidy_status')
return cls.create(scrobble_data)
@classmethod
def create_or_update_for_sport_event(
cls, event: "SportEvent", user_id: int, scrobble_data: dict
) -> "Scrobble":
scrobble_data['sport_event_id'] = event.id
scrobble = (
cls.objects.filter(
sport_event=event,
user_id=user_id,
played_to_completion=False,
)
.order_by('-modified')
.first()
)
if scrobble:
logger.debug(
f"Found existing scrobble for sport event {event}, updating",
{"scrobble_data": scrobble_data},
)
return cls.update(scrobble, scrobble_data)
logger.debug(
f"No existing scrobble for sport event {event}, creating",
{"scrobble_data": scrobble_data},
)
# If creating a new scrobble, we don't need status
scrobble_data.pop('jellyfin_status')
return cls.create(scrobble_data)
@classmethod
def update(cls, scrobble: "Scrobble", scrobble_data: dict) -> "Scrobble":
def update(self, scrobble_data: dict) -> "Scrobble":
# Status is a field we get from Mopidy, which refuses to poll us
scrobble_status = scrobble_data.pop('mopidy_status', None)
if not scrobble_status:
scrobble_status = scrobble_data.pop('jellyfin_status', None)
logger.debug(f"Scrobbling to {scrobble} with status {scrobble_status}")
scrobble.update_ticks(scrobble_data)
if self.percent_played < 100:
# Only worry about ticks if we haven't gotten to the end
self.update_ticks(scrobble_data)
# On stop, stop progress and send it to the check for completion
if scrobble_status == "stopped":
scrobble.stop()
self.stop()
# On pause, set is_paused and stop scrobbling
if scrobble_status == "paused":
scrobble.pause()
self.pause()
if scrobble_status == "resumed":
scrobble.resume()
self.resume()
for key, value in scrobble_data.items():
setattr(scrobble, key, value)
scrobble.save()
return scrobble
setattr(self, key, value)
self.save()
return self
@classmethod
def create(
@ -336,27 +326,27 @@ class Scrobble(TimeStampedModel):
def stop(self, force_finish=False) -> None:
if not self.in_progress:
logger.warning("Scrobble already stopped")
return
self.in_progress = False
self.save(update_fields=['in_progress'])
logger.info(f"{self.id} - {self.source}")
check_scrobble_for_finish(self, force_finish)
def pause(self) -> None:
print('Trying to pause it')
if self.is_paused:
logger.warning("Scrobble already paused")
logger.warning(f"{self.id} - already paused - {self.source}")
return
self.is_paused = True
self.save(update_fields=["is_paused"])
logger.info(f"{self.id} - pausing - {self.source}")
check_scrobble_for_finish(self)
def resume(self) -> None:
if self.is_paused or not self.in_progress:
self.is_paused = False
self.in_progress = True
logger.info(f"{self.id} - resuming - {self.source}")
return self.save(update_fields=["is_paused", "in_progress"])
logger.warning("Resume called but in progress or not paused")
def cancel(self) -> None:
check_scrobble_for_finish(self, force_finish=True)
@ -365,8 +355,8 @@ class Scrobble(TimeStampedModel):
def update_ticks(self, data) -> None:
self.playback_position_ticks = data.get("playback_position_ticks")
self.playback_position = data.get("playback_position")
logger.debug(
f"Updating scrobble ticks to {self.playback_position_ticks}"
logger.info(
f"{self.id} - {self.playback_position_ticks} - {self.source}"
)
self.save(
update_fields=['playback_position_ticks', 'playback_position']

View File

@ -50,9 +50,7 @@ def mopidy_scrobble_podcast(
scrobble = None
if episode:
scrobble = Scrobble.create_or_update_for_podcast_episode(
episode, user_id, mopidy_data
)
scrobble = Scrobble.create_or_update(episode, user_id, mopidy_data)
return scrobble
@ -90,23 +88,26 @@ def mopidy_scrobble_track(
track.musicbrainz_id = data_dict.get("musicbrainz_track_id")
track.save()
scrobble = Scrobble.create_or_update_for_track(track, user_id, mopidy_data)
scrobble = Scrobble.create_or_update(track, user_id, mopidy_data)
return scrobble
def create_jellyfin_scrobble_dict(data_dict: dict, user_id: int) -> dict:
def build_scrobble_dict(data_dict: dict, user_id: int) -> dict:
jellyfin_status = "resumed"
if data_dict.get("IsPaused"):
jellyfin_status = "paused"
if data_dict.get("NotificationType") == 'PlaybackStop':
elif data_dict.get("NotificationType") == 'PlaybackStop':
jellyfin_status = "stopped"
playback_ticks = data_dict.get("PlaybackPositionTicks", "")
if playback_ticks:
playback_ticks = playback_ticks // 10000
return {
"user_id": user_id,
"timestamp": parse(data_dict.get("UtcTimestamp")),
"playback_position_ticks": data_dict.get("PlaybackPositionTicks", "")
// 10000,
"playback_position_ticks": playback_ticks,
"playback_position": data_dict.get("PlaybackPosition", ""),
"source": data_dict.get("ClientName", "Vrobbler"),
"source_id": data_dict.get('MediaSourceId'),
@ -119,11 +120,22 @@ def jellyfin_scrobble_track(
) -> Optional[Scrobble]:
if not data_dict.get("Provider_musicbrainztrack", None):
# TODO we should be able to look up tracks via MB rather than error out
logger.error(
"No MBrainz Track ID received. This is likely because all metadata is bad, not scrobbling"
)
return
null_position_on_progress = (
data_dict.get("PlaybackPosition") == "00:00:00"
and data_dict.get("NotificationType") == "PlaybackProgress"
)
# Jellyfin has some race conditions with it's webhooks, these hacks fix some of them
if not data_dict.get("PlaybackPositionTicks") or null_position_on_progress:
logger.error("No playback position tick from Jellyfin, aborting")
return
artist_dict = {
'name': data_dict.get(JELLYFIN_POST_KEYS["ARTIST_NAME"], None),
'musicbrainz_id': data_dict.get(
@ -157,9 +169,13 @@ def jellyfin_scrobble_track(
)
track.save()
scrobble_dict = create_jellyfin_scrobble_dict(data_dict, user_id)
scrobble_dict = build_scrobble_dict(data_dict, user_id)
return Scrobble.create_or_update_for_track(track, user_id, scrobble_dict)
# A hack to make Jellyfin work more like Mopidy for music tracks
scrobble_dict["playback_position_ticks"] = 0
scrobble_dict["playback_position"] = ""
return Scrobble.create_or_update(track, user_id, scrobble_dict)
def jellyfin_scrobble_video(data_dict: dict, user_id: Optional[int]):
@ -170,9 +186,9 @@ def jellyfin_scrobble_video(data_dict: dict, user_id: Optional[int]):
return
video = Video.find_or_create(data_dict)
scrobble_dict = create_jellyfin_scrobble_dict(data_dict, user_id)
scrobble_dict = build_scrobble_dict(data_dict, user_id)
return Scrobble.create_or_update_for_video(video, user_id, scrobble_dict)
return Scrobble.create_or_update(video, user_id, scrobble_dict)
def manual_scrobble_video(data_dict: dict, user_id: Optional[int]):
@ -183,9 +199,9 @@ def manual_scrobble_video(data_dict: dict, user_id: Optional[int]):
return
video = Video.find_or_create(data_dict)
scrobble_dict = create_jellyfin_scrobble_dict(data_dict, user_id)
scrobble_dict = build_scrobble_dict(data_dict, user_id)
return Scrobble.create_or_update_for_video(video, user_id, scrobble_dict)
return Scrobble.create_or_update(video, user_id, scrobble_dict)
def manual_scrobble_event(data_dict: dict, user_id: Optional[int]):
@ -196,8 +212,6 @@ def manual_scrobble_event(data_dict: dict, user_id: Optional[int]):
return
event = SportEvent.find_or_create(data_dict)
scrobble_dict = create_jellyfin_scrobble_dict(data_dict, user_id)
scrobble_dict = build_scrobble_dict(data_dict, user_id)
return Scrobble.create_or_update_for_sport_event(
event, user_id, scrobble_dict
)
return Scrobble.create_or_update(event, user_id, scrobble_dict)

View File

@ -1,5 +1,11 @@
from rest_framework import serializers
from scrobbles.models import Scrobble
from scrobbles.models import Scrobble, AudioScrobblerTSVImport
class AudioScrobblerTSVImportSerializer(serializers.ModelSerializer):
class Meta:
model = AudioScrobblerTSVImport
fields = ('tsv_file',)
class ScrobbleSerializer(serializers.ModelSerializer):

View File

@ -19,9 +19,10 @@ def lookup_event_from_thesportsdb(event_id: str) -> dict:
return {}
league = {} # client.lookup_league(league_id=event.get('idLeague'))
event_type = "Game"
sport = Sport.objects.filter(thesportsdb_id=event.get('strSport')).first()
sport, _created = Sport.objects.get_or_create(
thesportsdb_id=event.get('strSport')
)
logger.debug(event)
data_dict = {
"ItemType": sport.default_event_type,
"Name": event.get('strEvent'),

View File

@ -0,0 +1,120 @@
import csv
import logging
from datetime import datetime
import pytz
from music.models import Album, Artist, Track
from scrobbles.models import Scrobble
logger = logging.getLogger(__name__)
def process_audioscrobbler_tsv_file(file_path, tz=None):
"""Takes a path to a file of TSV data and imports it as past scrobbles"""
new_scrobbles = []
if not tz:
tz = pytz.utc
with open(file_path) as infile:
source = 'Audioscrobbler File'
rows = csv.reader(infile, delimiter="\t")
source_id = ""
for row_num, row in enumerate(rows):
if row_num in [0, 1, 2]:
source_id += row[0] + "\n"
continue
if len(row) > 8:
logger.warning(
'Improper row length during Audioscrobbler import',
extra={'row': row},
)
continue
artist, artist_created = Artist.objects.get_or_create(name=row[0])
if artist_created:
logger.debug(f"Created artist {artist}")
else:
logger.debug(f"Found artist {artist}")
album = None
album_created = False
albums = Album.objects.filter(name=row[1])
if albums.count() == 1:
album = albums.first()
else:
for potential_album in albums:
if artist in album.artist_set.all():
album = potential_album
if not album:
album_created = True
album = Album.objects.create(name=row[1])
album.save()
album.artists.add(artist)
if album_created:
logger.debug(f"Created album {album}")
else:
logger.debug(f"Found album {album}")
track, track_created = Track.objects.get_or_create(
title=row[2],
artist=artist,
album=album,
)
if track_created:
logger.debug(f"Created track {track}")
else:
logger.debug(f"Found track {track}")
if track_created:
track.musicbrainz_id = row[7]
track.save()
timestamp = datetime.utcfromtimestamp(int(row[6])).replace(
tzinfo=tz
)
source = 'Audioscrobbler File'
new_scrobble = Scrobble(
timestamp=timestamp,
source=source,
source_id=source_id,
track=track,
played_to_completion=True,
in_progress=False,
)
existing = Scrobble.objects.filter(
timestamp=timestamp, track=track
).first()
if existing:
logger.debug(f"Skipping existing scrobble {new_scrobble}")
continue
logger.debug(f"Queued scrobble {new_scrobble} for creation")
new_scrobbles.append(new_scrobble)
created = Scrobble.objects.bulk_create(new_scrobbles)
logger.info(
f"Created {len(created)} scrobbles",
extra={'created_scrobbles': created},
)
return created
def undo_audioscrobbler_tsv_import(process_log, dryrun=True):
"""Accepts the log from a TSV import and removes the scrobbles"""
if not process_log:
logger.warning("No lines in process log found to undo")
return
for line_num, line in enumerate(process_log.split('\n')):
if line_num == 0:
continue
scrobble_id = line.split("\t")[0]
scrobble = Scrobble.objects.filter(id=scrobble_id).first()
if not scrobble:
logger.warning(f"Could not find scrobble {scrobble_id} to undo")
continue
logger.info(f"Removing scrobble {scrobble_id}")
if not dryrun:
scrobble.delete()

View File

@ -7,6 +7,12 @@ urlpatterns = [
path('', views.scrobble_endpoint, name='api-list'),
path('finish/<slug:uuid>', views.scrobble_finish, name='finish'),
path('cancel/<slug:uuid>', views.scrobble_cancel, name='cancel'),
path(
'upload/',
views.AudioScrobblerImportCreateView.as_view(),
name='audioscrobbler-file-upload',
),
path('jellyfin/', views.jellyfin_websocket, name='jellyfin-websocket'),
path('mopidy/', views.mopidy_websocket, name='mopidy-websocket'),
path('export/', views.export, name='export'),
]

View File

@ -67,18 +67,36 @@ def parse_mopidy_uri(uri: str) -> dict:
def check_scrobble_for_finish(
scrobble: "Scrobble", force_finish=False
scrobble: "Scrobble", force_to_100=False, force_finish=False
) -> None:
completion_percent = scrobble.media_obj.COMPLETION_PERCENT
if scrobble.percent_played >= completion_percent or force_finish:
logger.debug(f"Completion percent {completion_percent} met, finishing")
logger.info(f"{scrobble.id} {completion_percent} met, finishing")
if (
scrobble.playback_position_ticks
and scrobble.media_obj.run_time_ticks
and force_to_100
):
scrobble.playback_position_ticks = (
scrobble.media_obj.run_time_ticks
)
logger.info(
f"{scrobble.playback_position_ticks} set to {scrobble.media_obj.run_time_ticks}"
)
scrobble.in_progress = False
scrobble.is_paused = False
scrobble.played_to_completion = True
scrobble.save(
update_fields=["in_progress", "is_paused", "played_to_completion"]
update_fields=[
"in_progress",
"is_paused",
"played_to_completion",
'playback_position_ticks',
]
)
if scrobble.percent_played % 5 == 0:

View File

@ -1,26 +1,34 @@
import json
import logging
from datetime import datetime
import pytz
from django.conf import settings
from django.contrib.auth.mixins import LoginRequiredMixin
from django.db.models.fields import timezone
from django.http import HttpResponseRedirect
from django.urls import reverse
from django.http import FileResponse, HttpResponseRedirect, JsonResponse
from django.urls import reverse, reverse_lazy
from django.utils import timezone
from django.views.decorators.csrf import csrf_exempt
from django.views.generic import FormView
from django.views.generic.edit import CreateView
from django.views.generic.list import ListView
from rest_framework import status
from rest_framework.decorators import api_view, permission_classes
from rest_framework.decorators import (
api_view,
parser_classes,
permission_classes,
)
from rest_framework.parsers import MultiPartParser
from rest_framework.permissions import IsAuthenticated
from rest_framework.response import Response
from scrobbles.constants import (
JELLYFIN_AUDIO_ITEM_TYPES,
JELLYFIN_VIDEO_ITEM_TYPES,
)
from scrobbles.forms import ScrobbleForm
from scrobbles.forms import ExportScrobbleForm, ScrobbleForm
from scrobbles.imdb import lookup_video_from_imdb
from scrobbles.models import Scrobble
from scrobbles.models import AudioScrobblerTSVImport, Scrobble
from scrobbles.scrobblers import (
jellyfin_scrobble_track,
jellyfin_scrobble_video,
@ -29,7 +37,10 @@ from scrobbles.scrobblers import (
mopidy_scrobble_podcast,
mopidy_scrobble_track,
)
from scrobbles.serializers import ScrobbleSerializer
from scrobbles.serializers import (
AudioScrobblerTSVImportSerializer,
ScrobbleSerializer,
)
from scrobbles.thesportsdb import lookup_event_from_thesportsdb
from vrobbler.apps.music.aggregators import (
@ -38,6 +49,7 @@ from vrobbler.apps.music.aggregators import (
top_tracks,
week_of_scrobbles,
)
from vrobbler.apps.scrobbles.export import export_scrobbles
logger = logging.getLogger(__name__)
@ -87,6 +99,7 @@ class RecentScrobbleList(ListView):
data['counts'] = scrobble_counts(user)
data['imdb_form'] = ScrobbleForm
data['export_form'] = ExportScrobbleForm
return data
def get_queryset(self):
@ -118,7 +131,49 @@ class ManualScrobbleView(FormView):
if data_dict:
manual_scrobble_event(data_dict, self.request.user.id)
return HttpResponseRedirect(reverse("home"))
return HttpResponseRedirect(reverse("vrobbler-home"))
class JsonableResponseMixin:
"""
Mixin to add JSON support to a form.
Must be used with an object-based FormView (e.g. CreateView)
"""
def form_invalid(self, form):
response = super().form_invalid(form)
if self.request.accepts('text/html'):
return response
else:
return JsonResponse(form.errors, status=400)
def form_valid(self, form):
# We make sure to call the parent's form_valid() method because
# it might do some processing (in the case of CreateView, it will
# call form.save() for example).
response = super().form_valid(form)
if self.request.accepts('text/html'):
return response
else:
data = {
'pk': self.object.pk,
}
return JsonResponse(data)
class AudioScrobblerImportCreateView(
LoginRequiredMixin, JsonableResponseMixin, CreateView
):
model = AudioScrobblerTSVImport
fields = ['tsv_file']
template_name = 'scrobbles/upload_form.html'
success_url = reverse_lazy('vrobbler-home')
def form_valid(self, form):
self.object = form.save(commit=False)
self.object.user = self.request.user
self.object.save()
return HttpResponseRedirect(self.get_success_url())
@csrf_exempt
@ -136,6 +191,12 @@ def scrobble_endpoint(request):
def jellyfin_websocket(request):
data_dict = request.data
if (
data_dict['NotificationType'] == 'PlaybackProgress'
and data_dict['ItemType'] == 'Audio'
):
return Response({}, status=status.HTTP_304_NOT_MODIFIED)
# For making things easier to build new input processors
if getattr(settings, "DUMP_REQUEST_DATA", False):
json_data = json.dumps(data_dict, indent=4)
@ -182,6 +243,29 @@ def mopidy_websocket(request):
return Response({'scrobble_id': scrobble.id}, status=status.HTTP_200_OK)
@csrf_exempt
@permission_classes([IsAuthenticated])
@api_view(['POST'])
@parser_classes([MultiPartParser])
def import_audioscrobbler_file(request):
"""Takes a TSV file in the Audioscrobbler format, saves it and processes the
scrobbles.
"""
scrobbles_created = []
# tsv_file = request.FILES[0]
file_serializer = AudioScrobblerTSVImportSerializer(data=request.data)
if file_serializer.is_valid():
import_file = file_serializer.save()
return Response(
{'scrobble_ids': scrobbles_created}, status=status.HTTP_200_OK
)
else:
return Response(
file_serializer.errors, status=status.HTTP_400_BAD_REQUEST
)
@csrf_exempt
@permission_classes([IsAuthenticated])
@api_view(['GET'])
@ -215,3 +299,23 @@ def scrobble_cancel(request, uuid):
return Response(
{'id': scrobble.id, 'status': 'cancelled'}, status=status.HTTP_200_OK
)
@permission_classes([IsAuthenticated])
@api_view(['GET'])
def export(request):
format = request.GET.get('export_type', 'csv')
start = request.GET.get('start')
end = request.GET.get('end')
logger.debug(f"Exporting all scrobbles in format {format}")
temp_file, extension = export_scrobbles(
start_date=start, end_date=end, format=format
)
now = datetime.now()
filename = f"vrobbler-export-{str(now)}.{extension}"
response = FileResponse(open(temp_file, 'rb'))
response["Content-Disposition"] = f'attachment; filename="{filename}"'
return response

View File

@ -49,7 +49,12 @@ class Sport(TheSportsDbMixin):
# run_time_ticks = run_time_seconds * 1000
@property
def default_event_run_time_ticks(self):
return self.default_event_run_time * 1000
default_run_time = getattr(
settings, 'DEFAULT_EVENT_RUNTIME_SECONDS', 14400
)
if self.default_event_run_time:
default_run_time = self.default_event_run_time
return default_run_time * 1000
class League(TheSportsDbMixin):

View File

@ -34,6 +34,7 @@ class Series(TimeStampedModel):
class Video(ScrobblableMixin):
COMPLETION_PERCENT = getattr(settings, 'VIDEO_COMPLETION_PERCENT', 90)
SECONDS_TO_STALE = getattr(settings, 'VIDEO_SECONDS_TO_STALE', 14400)
class VideoType(models.TextChoices):
UNKNOWN = 'U', _('Unknown')
@ -91,10 +92,6 @@ class Video(ScrobblableMixin):
series, series_created = Series.objects.get_or_create(
name=series_name
)
if series_created:
logger.debug(f"Created new series {series}")
else:
logger.debug(f"Found series {series}")
video_dict['video_type'] = Video.VideoType.TV_EPISODE
video, created = cls.objects.get_or_create(**video_dict)
@ -119,11 +116,8 @@ class Video(ScrobblableMixin):
video_extra_dict["tv_series_id"] = series.id
if not video.run_time_ticks:
logger.debug(f"Created new video: {video}")
for key, value in video_extra_dict.items():
setattr(video, key, value)
video.save()
else:
logger.debug(f"Found video {video}")
return video

View File

@ -271,7 +271,6 @@
{% endblock %}
</div>
</div>
{% block extra_js %}
<script src="https://cdn.jsdelivr.net/npm/feather-icons@4.28.0/dist/feather.min.js" integrity="sha384-uO3SXW5IuS1ZpFPKugNNWqTZRRglnUJK6UAZ/gxOX80nxEkN9NcGZTftn6RzhGWE" crossorigin="anonymous"></script><script src="https://cdn.jsdelivr.net/npm/chart.js@2.9.4/dist/Chart.min.js" integrity="sha384-zNy6FEbO50N+Cg5wap8IKA4M/ZnLJgzc6w2NqACZaK0u0FXfOWRRJOnQtpZun8ha" crossorigin="anonymous"></script>
<script><!-- comment ------------------------------------------------->
/* globals Chart:false, feather:false */
@ -320,6 +319,7 @@
})()
</script>
{% block extra_js %}
{% endblock %}
</body>
</html>

View File

@ -7,19 +7,20 @@
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<h1 class="h2">Dashboard</h1>
<div class="btn-toolbar mb-2 mb-md-0">
{% if user.is_authenticated %}
<div class="btn-group me-2">
<button type="button" class="btn btn-sm btn-outline-secondary">Share</button>
<button type="button" class="btn btn-sm btn-outline-secondary">Export</button>
<button type="button" class="btn btn-sm btn-outline-secondary" data-bs-toggle="modal" data-bs-target="#importModal">Import</button>
<button type="button" class="btn btn-sm btn-outline-secondary" data-bs-toggle="modal" data-bs-target="#exportModal">Export</button>
</div>
{% endif %}
<div class="dropdown">
<button type="button" class="btn btn-sm btn-outline-secondary dropdown-toggle" id="graphDateButton" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
<button type="button" class="btn btn-sm btn-outline-secondary dropdown-toggle" id="graphDateButton" data-bs-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
<span data-feather="calendar"></span>
This week
</button>
<div class="dropdown-menu" aria-labelledby="graphDateButton">
<a class="dropdown-item" href="#">Action</a>
<a class="dropdown-item" href="#">Another action</a>
<a class="dropdown-item" href="#">Something else here</a>
<div class="dropdown-menu" data-bs-toggle="#graphDataChange" aria-labelledby="graphDateButton">
<a class="dropdown-item" href="#">This month</a>
<a class="dropdown-item" href="#">This year</a>
</div>
</div>
</div>
@ -179,7 +180,11 @@
{% for scrobble in object_list %}
<tr>
<td>{{scrobble.timestamp|naturaltime}}</td>
{% if scrobble.track.album.cover_image %}
<td><img src="{{scrobble.track.album.cover_image.url}}" width=50 height=50 style="border:1px solid black;" /></td>
{% else %}
<td>{{scrobble.track.album.name}}</td>
{% endif %}
<td>{{scrobble.track.title}}</td>
<td>{{scrobble.track.artist.name}}</td>
</tr>
@ -263,9 +268,65 @@
</div>
</div>
{% else %}
{% endif %}
</div>
</main>
<div class="modal fade" id="importModal" tabindex="-1" role="dialog" aria-labelledby="importModalLabel" aria-hidden="true">
<div class="modal-dialog" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="importModalLabel">Import scrobbles</h5>
<button type="button" class="close" data-bs-dismiss="modal" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
</div>
<form action="{% url 'audioscrobbler-file-upload' %}" method="post" enctype="multipart/form-data">
<div class="modal-body">
{% csrf_token %}
<div class="form-group">
<label for="tsv_file" class="col-form-label">Audioscrobbler TSV file:</label>
<input type="file" name="tsv_file" class="form-control" id="id_tsv_file">
</div>
</div>
<div class="modal-footer">
<button type="submit" class="btn btn-primary">Import</button>
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</form>
</div>
</div>
</div>
<div class="modal fade" id="exportModal" tabindex="-1" role="dialog" aria-labelledby="exportModalLabel" aria-hidden="true">
<div class="modal-dialog" role="document">
<div class="modal-content">
<div class="modal-header">
<h5 class="modal-title" id="exportModalLabel">Export scrobbles</h5>
<button type="button" class="close" data-bs-dismiss="modal" aria-label="Close">
<span aria-hidden="true">&times;</span>
</button>
</div>
<form action="{% url 'scrobbles:export' %}" method="get">
<div class="modal-body">
{% csrf_token %}
<div class="form-group">
{{export_form.as_div}}
</div>
</div>
<div class="modal-footer">
<button type="submit" class="btn btn-primary">Export</button>
<button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
</div>
</form>
</div>
</div>
</div>
{% endblock %}
{% block extra_js %}
<script>
$('#importModal').on('shown.bs.modal', function () { $('#importInput').trigger('focus') });
$('#exportModal').on('shown.bs.modal', function () { $('#exportInput').trigger('focus') });
</script>
{% endblock %}

View File

@ -0,0 +1,13 @@
{% extends "base.html" %}
{% block content %}
<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
<div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
<h1 class="h2">Manual scrobble</h1>
<form action="{% url 'audioscrobbler-file-upload' %}" method="post">
{% csrf_token %}
{{ form }}
<input type="submit" value="Submit">
</form>
</div>
</main>
{% endblock %}

View File

@ -19,9 +19,16 @@ urlpatterns = [
scrobbles_views.ManualScrobbleView.as_view(),
name='imdb-manual-scrobble',
),
path(
'manual/audioscrobbler/',
scrobbles_views.AudioScrobblerImportCreateView.as_view(),
name='audioscrobbler-file-upload',
),
path("", include(music_urls, namespace="music")),
path("", include(video_urls, namespace="videos")),
path("", scrobbles_views.RecentScrobbleList.as_view(), name="home"),
path(
"", scrobbles_views.RecentScrobbleList.as_view(), name="vrobbler-home"
),
]
if settings.DEBUG: