diff --git a/vrobbler/apps/scrobbles/admin.py b/vrobbler/apps/scrobbles/admin.py index 962349f..5a3833c 100644 --- a/vrobbler/apps/scrobbles/admin.py +++ b/vrobbler/apps/scrobbles/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from scrobbles.models import Scrobble +from scrobbles.models import AudioScrobblerTSVImport, Scrobble class ScrobbleInline(admin.TabularInline): @@ -10,6 +10,13 @@ class ScrobbleInline(admin.TabularInline): exclude = ('source_id', 'scrobble_log') +@admin.register(AudioScrobblerTSVImport) +class AudioScrobblerTSVImportAdmin(admin.ModelAdmin): + date_hierarchy = "created" + list_display = ("id", "tsv_file", "created") + ordering = ("-created",) + + @admin.register(Scrobble) class ScrobbleAdmin(admin.ModelAdmin): date_hierarchy = "timestamp" diff --git a/vrobbler/apps/scrobbles/forms.py b/vrobbler/apps/scrobbles/forms.py index 34bc6ce..e33cc16 100644 --- a/vrobbler/apps/scrobbles/forms.py +++ b/vrobbler/apps/scrobbles/forms.py @@ -1,5 +1,13 @@ from django import forms +from scrobbles.models import AudioScrobblerTSVImport + + +class UploadAudioscrobblerFileForm(forms.ModelForm): + class Meta: + model = AudioScrobblerTSVImport + fields = ('tsv_file',) + class ScrobbleForm(forms.Form): item_id = forms.CharField( diff --git a/vrobbler/apps/scrobbles/migrations/0012_audioscrobblertsvimport.py b/vrobbler/apps/scrobbles/migrations/0012_audioscrobblertsvimport.py new file mode 100644 index 0000000..8d89ddf --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0012_audioscrobblertsvimport.py @@ -0,0 +1,52 @@ +# Generated by Django 4.1.5 on 2023-02-03 19:50 + +from django.db import migrations, models +import django_extensions.db.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('scrobbles', '0011_chartrecord_user'), + ] + + operations = [ + migrations.CreateModel( + name='AudioScrobblerTSVImport', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ( + 'created', + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name='created' + ), + ), + ( + 'modified', + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name='modified' + ), + ), + ( + 'tsv_file', + models.FileField( + blank=True, + null=True, + upload_to='audioscrobbler-uploads/%Y/%m-%d/', + ), + ), + ], + options={ + 'get_latest_by': 'modified', + 'abstract': False, + }, + ), + ] diff --git a/vrobbler/apps/scrobbles/migrations/0013_audioscrobblertsvimport_processed_on.py b/vrobbler/apps/scrobbles/migrations/0013_audioscrobblertsvimport_processed_on.py new file mode 100644 index 0000000..546f65e --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0013_audioscrobblertsvimport_processed_on.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.5 on 2023-02-03 20:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scrobbles', '0012_audioscrobblertsvimport'), + ] + + operations = [ + migrations.AddField( + model_name='audioscrobblertsvimport', + name='processed_on', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index c904277..dc3887d 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -1,6 +1,5 @@ import calendar import logging -from datetime import timedelta from uuid import uuid4 from django.contrib.auth import get_user_model @@ -12,13 +11,38 @@ from podcasts.models import Episode from scrobbles.utils import check_scrobble_for_finish from sports.models import SportEvent from videos.models import Series, Video -from vrobbler.apps.profiles.utils import now_user_timezone logger = logging.getLogger(__name__) User = get_user_model() BNULL = {"blank": True, "null": True} +class AudioScrobblerTSVImport(TimeStampedModel): + tsv_file = models.FileField( + upload_to="audioscrobbler-uploads/%Y/%m-%d/", **BNULL + ) + processed_on = models.DateTimeField(**BNULL) + + def __str__(self): + return f"Audioscrobbler TSV upload: {self.tsv_file.path}" + + def save(self, **kwargs): + """On save, attempt to import the TSV file""" + + return super().save(**kwargs) + + def process(self, force=False): + from scrobbles.tsv import process_audioscrobbler_tsv_file + + if self.processed_on and not force: + logger.info(f"{self} already processed on {self.processed_on}") + return + + process_audioscrobbler_tsv_file(self.tsv_file.path) + self.processed_on = timezone.now() + self.save(update_fields=['processed_on']) + + class ChartRecord(TimeStampedModel): """Sort of like a materialized view for what we could dynamically generate, but would kill the DB as it gets larger. Collects time-based records diff --git a/vrobbler/apps/scrobbles/tsv.py b/vrobbler/apps/scrobbles/tsv.py new file mode 100644 index 0000000..e40baaf --- /dev/null +++ b/vrobbler/apps/scrobbles/tsv.py @@ -0,0 +1,98 @@ +import csv +import logging +from datetime import datetime + +import pytz +from music.models import Album, Artist, Track +from scrobbles.models import Scrobble + +logger = logging.getLogger(__name__) + + +def process_audioscrobbler_tsv_file(file_path): + """Takes a path to a file of TSV data and imports it as past scrobbles""" + new_scrobbles = [] + + with open(file_path) as infile: + source = 'Audioscrobbler File' + rows = csv.reader(infile, delimiter="\t") + + source_id = "" + for row_num, row in enumerate(rows): + if row_num in [0, 1, 2]: + source_id += row[0] + "\n" + continue + if len(row) > 8: + logger.warning( + 'Improper row length during Audioscrobbler import', + extra={'row': row}, + ) + continue + artist, artist_created = Artist.objects.get_or_create(name=row[0]) + if artist_created: + logger.debug(f"Created artist {artist}") + else: + logger.debug(f"Found artist {artist}") + + album = None + album_created = False + albums = Album.objects.filter(name=row[1]) + if albums.count() == 1: + album = albums.first() + else: + for potential_album in albums: + if artist in album.artist_set.all(): + album = potential_album + if not album: + album_created = True + album = Album.objects.create(name=row[1]) + album.save() + album.artists.add(artist) + + if album_created: + logger.debug(f"Created album {album}") + else: + logger.debug(f"Found album {album}") + + track, track_created = Track.objects.get_or_create( + title=row[2], + artist=artist, + album=album, + ) + + if track_created: + logger.debug(f"Created track {track}") + else: + logger.debug(f"Found track {track}") + + if track_created: + track.musicbrainz_id = row[7] + track.save() + + timestamp = datetime.utcfromtimestamp(int(row[6])).replace( + tzinfo=pytz.utc + ) + source = 'Audioscrobbler File' + + new_scrobble = Scrobble( + timestamp=timestamp, + source=source, + source_id=source_id, + track=track, + played_to_completion=True, + in_progress=False, + ) + existing = Scrobble.objects.filter( + timestamp=timestamp, track=track + ).first() + if existing: + logger.debug(f"Skipping existing scrobble {new_scrobble}") + continue + logger.debug(f"Queued scrobble {new_scrobble} for creation") + new_scrobbles.append(new_scrobble) + + created = Scrobble.objects.bulk_create(new_scrobbles) + logger.info( + f"Created {len(created)} scrobbles", + extra={'created_scrobbles': created}, + ) diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index 26bb77c..8476e99 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -18,7 +18,7 @@ from scrobbles.constants import ( JELLYFIN_AUDIO_ITEM_TYPES, JELLYFIN_VIDEO_ITEM_TYPES, ) -from scrobbles.forms import ScrobbleForm +from scrobbles.forms import ScrobbleForm, UploadAudioscrobblerFileForm from scrobbles.imdb import lookup_video_from_imdb from scrobbles.models import Scrobble from scrobbles.scrobblers import ( @@ -121,6 +121,11 @@ class ManualScrobbleView(FormView): return HttpResponseRedirect(reverse("home")) +class AudioScrobblerUploadView(FormView): + form_class = UploadAudioscrobblerFileForm + template_name = 'scrobbles/upload_form.html' + + @csrf_exempt @api_view(['GET']) def scrobble_endpoint(request): diff --git a/vrobbler/templates/scrobbles/upload_form.html b/vrobbler/templates/scrobbles/upload_form.html new file mode 100644 index 0000000..4e54ca0 --- /dev/null +++ b/vrobbler/templates/scrobbles/upload_form.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} +{% block content %} +
+
+

Manual scrobble

+
+ {% csrf_token %} + {{ form }} + +
+
+
+{% endblock %} diff --git a/vrobbler/urls.py b/vrobbler/urls.py index 2fe84a1..43af0da 100644 --- a/vrobbler/urls.py +++ b/vrobbler/urls.py @@ -19,6 +19,11 @@ urlpatterns = [ scrobbles_views.ManualScrobbleView.as_view(), name='imdb-manual-scrobble', ), + path( + 'manual/audioscrobbler/', + scrobbles_views.AudioScrobblerUploadView.as_view(), + name='audioscrobbler-file-upload', + ), path("", include(music_urls, namespace="music")), path("", include(video_urls, namespace="videos")), path("", scrobbles_views.RecentScrobbleList.as_view(), name="home"),