Clean up lastfm importing

vrobbler/apps/music/utils.py

@@ -1,3 +1,4 @@
+import re
 import logging
 
 from scrobbles.musicbrainz import (
@@ -12,18 +13,27 @@ from music.models import Artist, Album, Track
 
 
 def get_or_create_artist(name: str, mbid: str = None) -> Artist:
-    if 'feat.' in name:
+    artist = None
-        name = name.split('feat.')[0]
+    logger.debug(f'Got artist {name} and mbid: {mbid}')
-    if mbid:
-        artist, created = Artist.objects.get_or_create(musicbrainz_id=mbid)
-    else:
-        artist, created = Artist.objects.get_or_create(name=name)
 
-    if created or not mbid:
+    if 'feat.' in name.lower():
-        artist_dict = lookup_artist_from_mb(name)
+        name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip()
-        artist.musicbrainz_id = artist_dict["id"]
+    if 'featuring' in name.lower():
+        name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip()
+    artist_dict = lookup_artist_from_mb(name)
+    mbid = mbid or artist_dict['id']
+
+    logger.debug(f'Looking up artist {name} and mbid: {mbid}')
+    artist, created = Artist.objects.get_or_create(
+        name=name, musicbrainz_id=mbid
+    )
+
+    logger.debug(f"Cleaning artist {name} with {artist_dict['name']}")
+    # Clean up bad names in our DB with MB names
+    if artist.name != artist_dict['name']:
         artist.name = artist_dict["name"]
-        artist.save(update_fields=["musicbrainz_id", "name"])
+        artist.save(update_fields=["name"])
+
     return artist
 
 

vrobbler/apps/scrobbles/admin.py

@@ -24,7 +24,12 @@ class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
 @admin.register(LastFmImport)
 class LastFmImportAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
-    list_display = ("uuid", "created", "process_count", "processed_on")
+    list_display = (
+        "uuid",
+        "process_count",
+        "processed_finished",
+        "processing_started",
+    )
     ordering = ("-created",)
 
 

vrobbler/apps/scrobbles/lastfm.py

@@ -124,20 +124,33 @@ class LastFM:
         lfm_params['limit'] = None
 
         found_scrobbles = self.user.get_recent_tracks(**lfm_params)
+        # TOOD spin this out into a celery task over certain threshold of found scrobbles?
 
         for scrobble in found_scrobbles:
+            run_time = None
+            run_time_ticks = None
+            mbid = None
+            artist = None
+
             try:
                 run_time_ticks = scrobble.track.get_duration()
                 run_time = int(run_time_ticks / 1000)
-            except pylast.MalformedResponseError:
+                mbid = scrobble.track.get_mbid()
-                run_time_ticks = None
+                artist = scrobble.track.get_artist().name
-                run_time = None
+            except pylast.MalformedResponseError as e:
-                logger.warn(f"Track {scrobble.track} has no duration")
+                logger.warn(e)
+            except pylast.WSError as e:
+                logger.warn(
+                    "LastFM barfed trying to get the track for {scrobble.track}"
+                )
+
+            if not mbid or not artist:
+                logger.warn(f"Silly LastFM, bad data, bailing on {scrobble}")
+                continue
 
             timestamp = datetime.utcfromtimestamp(
                 int(scrobble.timestamp)
             ).replace(tzinfo=pytz.utc)
-            artist = scrobble.track.get_artist().name
 
             logger.info(f"{artist},{scrobble.track.title},{timestamp}")
             scrobbles.append(
@@ -145,7 +158,7 @@ class LastFM:
                     "artist": artist,
                     "album": scrobble.album,
                     "title": scrobble.track.title,
-                    "mbid": scrobble.track.get_mbid(),
+                    "mbid": mbid,
                     "run_time": run_time,
                     "run_time_ticks": run_time_ticks,
                     "timestamp": timestamp,

vrobbler/apps/scrobbles/migrations/0019_rename_processed_on_lastfmimport_processed_finished_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-02-16 17:13
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0018_lastfmimport'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='lastfmimport',
+            old_name='processed_on',
+            new_name='processed_finished',
+        ),
+        migrations.AddField(
+            model_name='lastfmimport',
+            name='processing_started',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -82,7 +82,8 @@ class AudioScrobblerTSVImport(TimeStampedModel):
 class LastFmImport(TimeStampedModel):
     user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
     uuid = models.UUIDField(editable=False, default=uuid4)
-    processed_on = models.DateTimeField(**BNULL)
+    processing_started = models.DateTimeField(**BNULL)
+    processed_finished = models.DateTimeField(**BNULL)
     process_log = models.TextField(**BNULL)
     process_count = models.IntegerField(**BNULL)
 
@@ -91,8 +92,10 @@ class LastFmImport(TimeStampedModel):
 
     def process(self, import_all=False):
         """Import scrobbles found on LastFM"""
-        if self.processed_on:
+        if self.processed_finished:
-            logger.info(f"{self} already processed on {self.processed_on}")
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
             return
 
         last_import = None
@@ -111,7 +114,10 @@ class LastFmImport(TimeStampedModel):
         lastfm = LastFM(self.user)
         last_processed = None
         if last_import:
-            last_processed = last_import.processed_on
+            last_processed = last_import.processed_finished
+
+        self.processing_started = timezone.now()
+        self.save(update_fields=['processing_started'])
 
         scrobbles = lastfm.import_from_lastfm(last_processed)
         self.process_log = ""
@@ -124,19 +130,22 @@ class LastFmImport(TimeStampedModel):
                 self.process_log += log_line
             self.process_count = len(scrobbles)
         else:
-            self.process_log = f"Created no new scrobbles"
             self.process_count = 0
 
-        self.processed_on = timezone.now()
+        self.processed_finished = timezone.now()
         self.save(
-            update_fields=['processed_on', 'process_count', 'process_log']
+            update_fields=[
+                'processed_finished',
+                'process_count',
+                'process_log',
+            ]
         )
 
     def undo(self, dryrun=False):
         """Undo import of scrobbles from LastFM"""
         LastFM.undo_lastfm_import(self.process_log, dryrun)
-        self.processed_on = None
+        self.processed_finished = None
-        self.save(update_fields=['processed_on'])
+        self.save(update_fields=['processed_finished'])
 
 
 class ChartRecord(TimeStampedModel):

vrobbler/apps/scrobbles/views.py

@@ -180,7 +180,7 @@ class AudioScrobblerImportCreateView(
 @api_view(['GET'])
 def lastfm_import(request):
     lfm_import, created = LastFmImport.objects.get_or_create(
-        user=request.user, processed_on__isnull=True
+        user=request.user, processed_finished__isnull=True
     )
 
     process_lastfm_import.delay(lfm_import.id)

vrobbler/templates/scrobbles/scrobble_list.html

@@ -240,9 +240,7 @@
                                     {% for scrobble in video_scrobble_list %}
                                     <tr>
                                         <td>{{scrobble.timestamp|naturaltime}}</td>
-                                        <td>{% if scrobble.video.tv_series
+                                        <td>{% if scrobble.video.tv_series %}S{{scrobble.video.season_number}}E{{scrobble.video.episode_number}} -{% endif %} {{scrobble.video.title}}</td>
-                                            %}S{{scrobble.video.season_number}}E{{scrobble.video.episode_number}} -{%
-                                            endif %} {{scrobble.video.title}}</td>
                                         <td>{% if scrobble.video.tv_series %}{{scrobble.video.tv_series}}{% endif %}
                                         </td>
                                     </tr>
@@ -367,4 +365,4 @@
     $('#importModal').on('shown.bs.modal', function () { $('#importInput').trigger('focus') });
     $('#exportModal').on('shown.bs.modal', function () { $('#exportInput').trigger('focus') });
 </script>
-{% endblock %}
+{% endblock %}