Bump version to 0.8.0

Here we add a model for holding Audioscrobbler imports and some code to
process the tab-separated files we get from Rockbox.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vrobbler"
-version = "0.7.5"
+version = "0.8.0"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 
@@ -16,7 +16,7 @@ djangorestframework = "^3.13.1"
 Markdown = "^3.3.6"
 django-filter = "^21.1"
 Pillow = "^9.0.1"
-psycopg2 = {version = "^2.9.3", extras = ["production"]}
+psycopg2 = "^2.9.3"
 dj-database-url = "^0.5.0"
 django-mathfilters = "^1.0.0"
 django-allauth = "^0.50.0"

todos.org

@@ -9,7 +9,7 @@ When we get artwork from Musicbrianz, and it's not found, we should check for
 release groups as well. This will stop issues with missing artwork because of
 obscure MB release matches.
 
-* DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion perecnt :bug:
+* DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion percent :bug:
 CLOSED: [2023-01-30 Mon 18:31]
 :LOGBOOK:
 CLOCK: [2023-01-30 Mon 18:00]--[2023-01-30 Mon 18:31] =>  0:31
@@ -26,6 +26,33 @@ as complete for the following conditions:
 
 But if we keep listening beyond 90, we should basically ignore updates (or just
 update the existing scrobble)
+* DONE [#A] Add support for Audioscrobbler tab-separated file uploads :improvement:
+CLOSED: [2023-02-03 Fri 16:52]
+
+An example of the format:
+#+begin_src csv
+,
+#AUDIOSCROBBLER/1.1
+#TZ/UNKNOWN
+#CLIENT/Rockbox sansaclipplus $Revision$
+75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494944	64ff5f53-d187-4512-827e-7606c69e66ff
+75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494990	64ff5f53-d187-4512-827e-7606c69e66ff
+311	311	Down	1	173	S	1740495003	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Down	1	173	L	1740495049	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Down	1	173	L	1740495113	00476c23-fd9e-464b-9b27-a62d69f3d4f4
+311	311	Random	2	187	S	1740495190	530c09f3-46fe-4d90-b11f-7b63bcb4b373
+311	311	Random	2	187	L	1740495194	530c09f3-46fe-4d90-b11f-7b63bcb4b373
+311	311	Jackolantern’s Weather	3	204	L	1740495382	cc3b2dec-5d99-47ea-8930-20bf258be4ea
+311	311	All Mixed Up	4	182	L	1740495586	980a78b5-5bdd-4f50-9e3a-e13261e2817b
+311	311	Hive	5	179	L	1740495768	18f6dc98-d3a2-4f81-b967-97359d14c68c
+311	311	Guns (Are for Pussies)	6	137	L	1740495948	5e97ed9f-c8cc-4282-9cbe-f8e17aee5128
+311	311	Misdirected Hostility	7	179	S	1740496085	61ff2c1a-fc9c-44c3-8da1-5e50a44245af
+,
+#+end_src
+* TODO [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
+
+Given a UUID from musicbrainz, we should be able to scrobble an album or
+individual track.
 
 * TODO [#A] Add django-storage to store files on S3 :improvement:
 * TODO [#B] Adjust cancel/finish task to use javascript to submit :improvement:
@@ -349,3 +376,5 @@ has to re-populate when the server restarts.
 
 An example:
 https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
+
+* TODO When updating musicbrainz IDs, clear and run fetch artwrok :improvement:

vrobbler/apps/scrobbles/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-from scrobbles.models import Scrobble
+from scrobbles.models import AudioScrobblerTSVImport, Scrobble
 
 
 class ScrobbleInline(admin.TabularInline):
@@ -10,6 +10,13 @@ class ScrobbleInline(admin.TabularInline):
     exclude = ('source_id', 'scrobble_log')
 
 
+@admin.register(AudioScrobblerTSVImport)
+class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("uuid", "created", "process_count", "tsv_file")
+    ordering = ("-created",)
+
+
 @admin.register(Scrobble)
 class ScrobbleAdmin(admin.ModelAdmin):
     date_hierarchy = "timestamp"

vrobbler/apps/scrobbles/forms.py

@@ -1,5 +1,13 @@
 from django import forms
 
+from scrobbles.models import AudioScrobblerTSVImport
+
+
+class UploadAudioscrobblerFileForm(forms.ModelForm):
+    class Meta:
+        model = AudioScrobblerTSVImport
+        fields = ('tsv_file',)
+
 
 class ScrobbleForm(forms.Form):
     item_id = forms.CharField(

vrobbler/apps/scrobbles/imdb.py

@@ -1,9 +1,7 @@
 import logging
-from typing import Optional
 from django.utils import timezone
 
 from imdb import Cinemagoer
-from videos.models import Video
 
 imdb_client = Cinemagoer()
 

vrobbler/apps/scrobbles/migrations/0012_audioscrobblertsvimport.py

@@ -0,0 +1,52 @@
+# Generated by Django 4.1.5 on 2023-02-03 19:50
+
+from django.db import migrations, models
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0011_chartrecord_user'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AudioScrobblerTSVImport',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'tsv_file',
+                    models.FileField(
+                        blank=True,
+                        null=True,
+                        upload_to='audioscrobbler-uploads/%Y/%m-%d/',
+                    ),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0013_audioscrobblertsvimport_processed_on.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-03 20:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0012_audioscrobblertsvimport'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='processed_on',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0014_audioscrobblertsvimport_process_log.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-03 22:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0013_audioscrobblertsvimport_processed_on'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='process_log',
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0015_audioscrobblertsvimport_uuid_and_more.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.1.5 on 2023-02-03 23:36
+
+from django.db import migrations, models
+import scrobbles.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0014_audioscrobblertsvimport_process_log'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='uuid',
+            field=models.UUIDField(default=uuid.uuid4, editable=False),
+        ),
+        migrations.AlterField(
+            model_name='audioscrobblertsvimport',
+            name='tsv_file',
+            field=models.FileField(
+                blank=True,
+                null=True,
+                upload_to=scrobbles.models.AudioScrobblerTSVImport.get_path,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0016_audioscrobblertsvimport_process_count.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-03 23:44
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0015_audioscrobblertsvimport_uuid_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='process_count',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -1,6 +1,5 @@
 import calendar
 import logging
-from datetime import timedelta
 from uuid import uuid4
 
 from django.contrib.auth import get_user_model
@@ -12,13 +11,59 @@ from podcasts.models import Episode
 from scrobbles.utils import check_scrobble_for_finish
 from sports.models import SportEvent
 from videos.models import Series, Video
-from vrobbler.apps.profiles.utils import now_user_timezone
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
 BNULL = {"blank": True, "null": True}
 
 
+class AudioScrobblerTSVImport(TimeStampedModel):
+    def get_path(instance, filename):
+        extension = filename.split('.')[-1]
+        uuid = instance.uuid
+        return f'audioscrobbler-uploads/{uuid}.{extension}'
+
+    uuid = models.UUIDField(editable=False, default=uuid4)
+    tsv_file = models.FileField(upload_to=get_path, **BNULL)
+    processed_on = models.DateTimeField(**BNULL)
+    process_log = models.TextField(**BNULL)
+    process_count = models.IntegerField(**BNULL)
+
+    def __str__(self):
+        if self.tsv_file:
+            return f"Audioscrobbler TSV upload: {self.tsv_file.path}"
+        return f"Audioscrobbler TSV upload {self.id}"
+
+    def save(self, **kwargs):
+        """On save, attempt to import the TSV file"""
+        super().save(**kwargs)
+        self.process()
+        return
+
+    def process(self, force=False):
+        from scrobbles.tsv import process_audioscrobbler_tsv_file
+
+        if self.processed_on and not force:
+            logger.info(f"{self} already processed on {self.processed_on}")
+            return
+
+        scrobbles = process_audioscrobbler_tsv_file(self.tsv_file.path)
+        if scrobbles:
+            self.process_log = f"Created {len(scrobbles)} scrobbles"
+            for scrobble in scrobbles:
+                scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}\t"
+                self.process_log += f"\n{scrobble_str}"
+            self.process_count = len(scrobbles)
+        else:
+            self.process_log = f"Created no new scrobbles"
+            self.process_count = 0
+
+        self.processed_on = timezone.now()
+        self.save(
+            update_fields=['processed_on', 'process_count', 'process_log']
+        )
+
+
 class ChartRecord(TimeStampedModel):
     """Sort of like a materialized view for what we could dynamically generate,
     but would kill the DB as it gets larger. Collects time-based records

vrobbler/apps/scrobbles/serializers.py

@@ -1,5 +1,11 @@
 from rest_framework import serializers
-from scrobbles.models import Scrobble
+from scrobbles.models import Scrobble, AudioScrobblerTSVImport
+
+
+class AudioScrobblerTSVImportSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = AudioScrobblerTSVImport
+        fields = ('tsv_file',)
 
 
 class ScrobbleSerializer(serializers.ModelSerializer):

vrobbler/apps/scrobbles/tsv.py

@@ -0,0 +1,99 @@
+import csv
+import logging
+from datetime import datetime
+
+import pytz
+from music.models import Album, Artist, Track
+from scrobbles.models import Scrobble
+
+logger = logging.getLogger(__name__)
+
+
+def process_audioscrobbler_tsv_file(file_path):
+    """Takes a path to a file of TSV data and imports it as past scrobbles"""
+    new_scrobbles = []
+
+    with open(file_path) as infile:
+        source = 'Audioscrobbler File'
+        rows = csv.reader(infile, delimiter="\t")
+
+        source_id = ""
+        for row_num, row in enumerate(rows):
+            if row_num in [0, 1, 2]:
+                source_id += row[0] + "\n"
+                continue
+            if len(row) > 8:
+                logger.warning(
+                    'Improper row length during Audioscrobbler import',
+                    extra={'row': row},
+                )
+                continue
+            artist, artist_created = Artist.objects.get_or_create(name=row[0])
+            if artist_created:
+                logger.debug(f"Created artist {artist}")
+            else:
+                logger.debug(f"Found artist {artist}")
+
+            album = None
+            album_created = False
+            albums = Album.objects.filter(name=row[1])
+            if albums.count() == 1:
+                album = albums.first()
+            else:
+                for potential_album in albums:
+                    if artist in album.artist_set.all():
+                        album = potential_album
+            if not album:
+                album_created = True
+                album = Album.objects.create(name=row[1])
+                album.save()
+                album.artists.add(artist)
+
+            if album_created:
+                logger.debug(f"Created album {album}")
+            else:
+                logger.debug(f"Found album {album}")
+
+            track, track_created = Track.objects.get_or_create(
+                title=row[2],
+                artist=artist,
+                album=album,
+            )
+
+            if track_created:
+                logger.debug(f"Created track {track}")
+            else:
+                logger.debug(f"Found track {track}")
+
+            if track_created:
+                track.musicbrainz_id = row[7]
+                track.save()
+
+            timestamp = datetime.utcfromtimestamp(int(row[6])).replace(
+                tzinfo=pytz.utc
+            )
+            source = 'Audioscrobbler File'
+
+            new_scrobble = Scrobble(
+                timestamp=timestamp,
+                source=source,
+                source_id=source_id,
+                track=track,
+                played_to_completion=True,
+                in_progress=False,
+            )
+            existing = Scrobble.objects.filter(
+                timestamp=timestamp, track=track
+            ).first()
+            if existing:
+                logger.debug(f"Skipping existing scrobble {new_scrobble}")
+                continue
+            logger.debug(f"Queued scrobble {new_scrobble} for creation")
+            new_scrobbles.append(new_scrobble)
+
+        created = Scrobble.objects.bulk_create(new_scrobbles)
+        logger.info(
+            f"Created {len(created)} scrobbles",
+            extra={'created_scrobbles': created},
+        )
+        return created

vrobbler/apps/scrobbles/urls.py

@@ -7,6 +7,11 @@ urlpatterns = [
     path('', views.scrobble_endpoint, name='api-list'),
     path('finish/<slug:uuid>', views.scrobble_finish, name='finish'),
     path('cancel/<slug:uuid>', views.scrobble_cancel, name='cancel'),
+    path(
+        'audioscrobbler-file-upload/',
+        views.import_audioscrobbler_file,
+        name='audioscrobbler-file-upload',
+    ),
     path('jellyfin/', views.jellyfin_websocket, name='jellyfin-websocket'),
     path('mopidy/', views.mopidy_websocket, name='mopidy-websocket'),
 ]

vrobbler/apps/scrobbles/views.py

@@ -11,14 +11,19 @@ from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import FormView
 from django.views.generic.list import ListView
 from rest_framework import status
-from rest_framework.decorators import api_view, permission_classes
+from rest_framework.decorators import (
+    api_view,
+    parser_classes,
+    permission_classes,
+)
+from rest_framework.parsers import MultiPartParser
 from rest_framework.permissions import IsAuthenticated
 from rest_framework.response import Response
 from scrobbles.constants import (
     JELLYFIN_AUDIO_ITEM_TYPES,
     JELLYFIN_VIDEO_ITEM_TYPES,
 )
-from scrobbles.forms import ScrobbleForm
+from scrobbles.forms import ScrobbleForm, UploadAudioscrobblerFileForm
 from scrobbles.imdb import lookup_video_from_imdb
 from scrobbles.models import Scrobble
 from scrobbles.scrobblers import (
@@ -29,7 +34,10 @@ from scrobbles.scrobblers import (
     mopidy_scrobble_podcast,
     mopidy_scrobble_track,
 )
-from scrobbles.serializers import ScrobbleSerializer
+from scrobbles.serializers import (
+    AudioScrobblerTSVImportSerializer,
+    ScrobbleSerializer,
+)
 from scrobbles.thesportsdb import lookup_event_from_thesportsdb
 
 from vrobbler.apps.music.aggregators import (
@@ -121,6 +129,11 @@ class ManualScrobbleView(FormView):
         return HttpResponseRedirect(reverse("home"))
 
 
+class AudioScrobblerUploadView(FormView):
+    form_class = UploadAudioscrobblerFileForm
+    template_name = 'scrobbles/upload_form.html'
+
+
 @csrf_exempt
 @api_view(['GET'])
 def scrobble_endpoint(request):
@@ -182,6 +195,29 @@ def mopidy_websocket(request):
     return Response({'scrobble_id': scrobble.id}, status=status.HTTP_200_OK)
 
 
+@csrf_exempt
+@permission_classes([IsAuthenticated])
+@api_view(['POST'])
+@parser_classes([MultiPartParser])
+def import_audioscrobbler_file(request):
+    """Takes a TSV file in the Audioscrobbler format, saves it and processes the
+    scrobbles.
+    """
+    scrobbles_created = []
+    # tsv_file = request.FILES[0]
+
+    file_serializer = AudioScrobblerTSVImportSerializer(data=request.data)
+    if file_serializer.is_valid():
+        import_file = file_serializer.save()
+        return Response(
+            {'scrobble_ids': scrobbles_created}, status=status.HTTP_200_OK
+        )
+    else:
+        return Response(
+            file_serializer.errors, status=status.HTTP_400_BAD_REQUEST
+        )
+
+
 @csrf_exempt
 @permission_classes([IsAuthenticated])
 @api_view(['GET'])

vrobbler/templates/scrobbles/scrobble_list.html

@@ -179,7 +179,11 @@
                             {% for scrobble in object_list %}
                             <tr>
                                 <td>{{scrobble.timestamp|naturaltime}}</td>
+                                {% if scrobble.track.album.cover_image %}
                                 <td><img src="{{scrobble.track.album.cover_image.url}}" width=50 height=50 style="border:1px solid black;" /></td>
+                                {% else %}
+                                <td>{{scrobble.track.album.name}}</td>
+                                {% endif %}
                                 <td>{{scrobble.track.title}}</td>
                                 <td>{{scrobble.track.artist.name}}</td>
                             </tr>

vrobbler/templates/scrobbles/upload_form.html

@@ -0,0 +1,13 @@
+{% extends "base.html" %}
+{% block content %}
+<main class="col-md-4 ms-sm-auto col-lg-10 px-md-4">
+    <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pt-3 pb-2 mb-3 border-bottom">
+        <h1 class="h2">Manual scrobble</h1>
+        <form action="{% url 'audioscrobbler-file-upload' %}" method="post">
+            {% csrf_token %}
+            {{ form }}
+            <input type="submit" value="Submit">
+        </form>
+    </div>
+</main>
+{% endblock %}

vrobbler/urls.py

@@ -19,6 +19,11 @@ urlpatterns = [
         scrobbles_views.ManualScrobbleView.as_view(),
         name='imdb-manual-scrobble',
     ),
+    path(
+        'manual/audioscrobbler/',
+        scrobbles_views.AudioScrobblerUploadView.as_view(),
+        name='audioscrobbler-file-upload',
+    ),
     path("", include(music_urls, namespace="music")),
     path("", include(video_urls, namespace="videos")),
     path("", scrobbles_views.RecentScrobbleList.as_view(), name="home"),