Add import of Audioscrobbler files

Here we add a model for holding Audioscrobbler imports and some code to
process the tab-separated files we get from Rockbox.

vrobbler/apps/scrobbles/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-from scrobbles.models import Scrobble
+from scrobbles.models import AudioScrobblerTSVImport, Scrobble
 
 
 class ScrobbleInline(admin.TabularInline):
@@ -10,6 +10,13 @@ class ScrobbleInline(admin.TabularInline):
     exclude = ('source_id', 'scrobble_log')
 
 
+@admin.register(AudioScrobblerTSVImport)
+class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("id", "tsv_file", "created")
+    ordering = ("-created",)
+
+
 @admin.register(Scrobble)
 class ScrobbleAdmin(admin.ModelAdmin):
     date_hierarchy = "timestamp"

vrobbler/apps/scrobbles/forms.py

@@ -1,5 +1,13 @@
 from django import forms
 
+from scrobbles.models import AudioScrobblerTSVImport
+
+
+class UploadAudioscrobblerFileForm(forms.ModelForm):
+    class Meta:
+        model = AudioScrobblerTSVImport
+        fields = ('tsv_file',)
+
 
 class ScrobbleForm(forms.Form):
     item_id = forms.CharField(

vrobbler/apps/scrobbles/migrations/0012_audioscrobblertsvimport.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.1.5 on 2023-02-03 19:50
+
+from django.db import migrations, models
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0011_chartrecord_user'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AudioScrobblerTSVImport',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'tsv_file',
+                    models.FileField(
+                        blank=True,
+                        null=True,
+                        upload_to='audioscrobbler-uploads/%Y/%m-%d/',
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0013_audioscrobblertsvimport_processed_on.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-03 20:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0012_audioscrobblertsvimport'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='processed_on',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -1,6 +1,5 @@
 import calendar
 import logging
-from datetime import timedelta
 from uuid import uuid4
 
 from django.contrib.auth import get_user_model
@@ -12,13 +11,38 @@ from podcasts.models import Episode
 from scrobbles.utils import check_scrobble_for_finish
 from sports.models import SportEvent
 from videos.models import Series, Video
-from vrobbler.apps.profiles.utils import now_user_timezone
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
 BNULL = {"blank": True, "null": True}
 
 
+class AudioScrobblerTSVImport(TimeStampedModel):
+    tsv_file = models.FileField(
+        upload_to="audioscrobbler-uploads/%Y/%m-%d/", **BNULL
+    )
+    processed_on = models.DateTimeField(**BNULL)
+
+    def __str__(self):
+        return f"Audioscrobbler TSV upload: {self.tsv_file.path}"
+
+    def save(self, **kwargs):
+        """On save, attempt to import the TSV file"""
+
+        return super().save(**kwargs)
+
+    def process(self, force=False):
+        from scrobbles.tsv import process_audioscrobbler_tsv_file
+
+        if self.processed_on and not force:
+            logger.info(f"{self} already processed on {self.processed_on}")
+            return
+
+        process_audioscrobbler_tsv_file(self.tsv_file.path)
+        self.processed_on = timezone.now()
+        self.save(update_fields=['processed_on'])
+
+
 class ChartRecord(TimeStampedModel):
     """Sort of like a materialized view for what we could dynamically generate,
     but would kill the DB as it gets larger. Collects time-based records

vrobbler/apps/scrobbles/tsv.py

@@ -0,0 +1,98 @@
+import csv
+import logging
+from datetime import datetime
+
+import pytz
+from music.models import Album, Artist, Track
+from scrobbles.models import Scrobble
+
+logger = logging.getLogger(__name__)
+
+
+def process_audioscrobbler_tsv_file(file_path):
+    """Takes a path to a file of TSV data and imports it as past scrobbles"""
+    new_scrobbles = []
+
+    with open(file_path) as infile:
+        source = 'Audioscrobbler File'
+        rows = csv.reader(infile, delimiter="\t")
+
+        source_id = ""
+        for row_num, row in enumerate(rows):
+            if row_num in [0, 1, 2]:
+                source_id += row[0] + "\n"
+                continue
+            if len(row) > 8:
+                logger.warning(
+                    'Improper row length during Audioscrobbler import',
+                    extra={'row': row},
+                )
+                continue
+            artist, artist_created = Artist.objects.get_or_create(name=row[0])
+            if artist_created:
+                logger.debug(f"Created artist {artist}")
+            else:
+                logger.debug(f"Found artist {artist}")
+
+            album = None
+            album_created = False
+            albums = Album.objects.filter(name=row[1])
+            if albums.count() == 1:
+                album = albums.first()
+            else:
+                for potential_album in albums:
+                    if artist in album.artist_set.all():
+                        album = potential_album
+            if not album:
+                album_created = True
+                album = Album.objects.create(name=row[1])
+                album.save()
+                album.artists.add(artist)
+
+            if album_created:
+                logger.debug(f"Created album {album}")
+            else:
+                logger.debug(f"Found album {album}")
+
+            track, track_created = Track.objects.get_or_create(
+                title=row[2],
+                artist=artist,
+                album=album,
+            )
+
+            if track_created:
+                logger.debug(f"Created track {track}")
+            else:
+                logger.debug(f"Found track {track}")
+
+            if track_created:
+                track.musicbrainz_id = row[7]
+                track.save()
+
+            timestamp = datetime.utcfromtimestamp(int(row[6])).replace(
+                tzinfo=pytz.utc
+            )
+            source = 'Audioscrobbler File'
+
+            new_scrobble = Scrobble(
+                timestamp=timestamp,
+                source=source,
+                source_id=source_id,
+                track=track,
+                played_to_completion=True,
+                in_progress=False,
+            )
+            existing = Scrobble.objects.filter(
+                timestamp=timestamp, track=track
+            ).first()
+            if existing:
+                logger.debug(f"Skipping existing scrobble {new_scrobble}")
+                continue
+            logger.debug(f"Queued scrobble {new_scrobble} for creation")
+            new_scrobbles.append(new_scrobble)
+
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={'created_scrobbles': created},
+        )

vrobbler/apps/scrobbles/views.py

@@ -18,7 +18,7 @@ from scrobbles.constants import (
     JELLYFIN_AUDIO_ITEM_TYPES,
     JELLYFIN_VIDEO_ITEM_TYPES,
 )
-from scrobbles.forms import ScrobbleForm
+from scrobbles.forms import ScrobbleForm, UploadAudioscrobblerFileForm
 from scrobbles.imdb import lookup_video_from_imdb
 from scrobbles.models import Scrobble
 from scrobbles.scrobblers import (
@@ -121,6 +121,11 @@ class ManualScrobbleView(FormView):
         return HttpResponseRedirect(reverse("home"))
 
 
+class AudioScrobblerUploadView(FormView):
+    form_class = UploadAudioscrobblerFileForm
+    template_name = 'scrobbles/upload_form.html'
+
+
 @csrf_exempt
 @api_view(['GET'])
 def scrobble_endpoint(request):

vrobbler/templates/scrobbles/upload_form.html

@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+{% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">Manual scrobble</h1>
+        <form action="{% url 'audioscrobbler-file-upload' %}" method="post">
+            {% csrf_token %}
+            {{ form }}
+            <input type="submit" value="Submit">
+        </form>
+    </div>
+</main>
+{% endblock %}

vrobbler/urls.py

@@ -19,6 +19,11 @@ urlpatterns = [
         scrobbles_views.ManualScrobbleView.as_view(),
         name='imdb-manual-scrobble',
     ),
+    path(
+        'manual/audioscrobbler/',
+        scrobbles_views.AudioScrobblerUploadView.as_view(),
+        name='audioscrobbler-file-upload',
+    ),
     path("", include(music_urls, namespace="music")),
     path("", include(video_urls, namespace="videos")),
     path("", scrobbles_views.RecentScrobbleList.as_view(), name="home"),