[importers] Add error_log to surface import errors

vrobbler/apps/birds/admin.py

@@ -26,5 +26,5 @@ class BirdingLocationAdmin(admin.ModelAdmin):
 @admin.register(BirdingCSVImport)
 class BirdingCSVImportAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
-    list_display = ("uuid", "process_count", "processed_finished", "processing_started")
+    list_display = ("uuid", "process_count", "processed_finished", "processing_started", "error_log")
     ordering = ("-created",)

vrobbler/apps/birds/importer.py

@@ -64,7 +64,7 @@ def parse_int(value):
         return None
 
 
-def import_birding_csv(file_path, user_id):
+def import_birding_csv(file_path, user_id, record_error=None):
     user = User.objects.get(id=user_id)
     new_scrobbles = []
 
@@ -83,11 +83,17 @@ def import_birding_csv(file_path, user_id):
 
     for (location_str, date_str, time_str), sighting_rows in groups.items():
         if not location_str:
-            logger.warning("Skipping rows with no location")
+            msg = "Skipping rows with no location"
+            logger.warning(msg)
+            if record_error:
+                record_error(msg)
             continue
 
         timestamp = parse_timestamp(date_str, time_str)
         if not timestamp:
+            msg = f"Could not parse date/time: {date_str} {time_str}"
+            if record_error:
+                record_error(msg)
             continue
 
         timestamp = user.profile.get_timestamp_with_tz(timestamp)

vrobbler/apps/birds/migrations/0003_birdingcsvimport_error_log.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.29 on 2026-06-08 14:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("birds", "0002_birdingcsvimport"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="birdingcsvimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/birds/models.py

@@ -216,6 +216,7 @@ class BirdingCSVImport(TimeStampedModel):
     processed_finished = models.DateTimeField(**BNULL)
     process_log = models.TextField(**BNULL)
     process_count = models.IntegerField(**BNULL)
+    error_log = models.TextField(**BNULL)
     csv_file = models.FileField(upload_to="birding-csv-uploads/", **BNULL)
 
     class Meta:
@@ -269,6 +270,14 @@ class BirdingCSVImport(TimeStampedModel):
         self.process_count = len(scrobbles)
         self.save(update_fields=["process_log", "process_count"])
 
+    def record_error(self, error_message):
+        log_line = f"{timezone.now().isoformat()}: {error_message}"
+        if self.error_log:
+            self.error_log += "\n" + log_line
+        else:
+            self.error_log = log_line
+        self.save(update_fields=["error_log"])
+
     def scrobbles(self):
         from scrobbles.models import Scrobble
 
@@ -287,6 +296,13 @@ class BirdingCSVImport(TimeStampedModel):
         from birds.importer import import_birding_csv
 
         self.mark_started()
-        scrobbles = import_birding_csv(self.upload_file_path, self.user_id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_birding_csv(
+                self.upload_file_path, self.user_id, record_error=self.record_error
+            )
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()

vrobbler/apps/scrobbles/admin.py

@@ -56,6 +56,7 @@ class ImportBaseAdmin(admin.ModelAdmin):
         "process_count",
         "processed_finished",
         "processing_started",
+        "error_log",
     )
     ordering = ("-created",)
 

vrobbler/apps/scrobbles/migrations/0090_audioscrobblertsvimport_error_log_and_more.py

@@ -0,0 +1,53 @@
+# Generated by Django 4.2.29 on 2026-06-08 14:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0089_favoritemedia"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="audioscrobblertsvimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="bgstatsimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="ebirdcsvimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="koreaderimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="lastfmimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="retroarchimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="scalecsvimport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="trailgpximport",
+            name="error_log",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -74,6 +74,7 @@ class BaseFileImportMixin(TimeStampedModel):
     processed_finished = models.DateTimeField(**BNULL)
     process_log = models.TextField(**BNULL)
     process_count = models.IntegerField(**BNULL)
+    error_log = models.TextField(**BNULL)
 
     class Meta:
         abstract = True
@@ -158,6 +159,14 @@ class BaseFileImportMixin(TimeStampedModel):
         self.process_count = len(scrobbles)
         self.save(update_fields=["process_log", "process_count"])
 
+    def record_error(self, error_message):
+        log_line = f"{timezone.now().isoformat()}: {error_message}"
+        if self.error_log:
+            self.error_log += "\n" + log_line
+        else:
+            self.error_log = log_line
+        self.save(update_fields=["error_log"])
+
     @property
     def upload_file_path(self):
         raise NotImplementedError
@@ -213,9 +222,14 @@ class KoReaderImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = process_koreader_sqlite_file(self.upload_file_path, self.user.id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = process_koreader_sqlite_file(self.upload_file_path, self.user.id)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class AudioScrobblerTSVImport(BaseFileImportMixin):
@@ -255,10 +269,14 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-
-        scrobbles = import_audioscrobbler_tsv_file(self.upload_file_path, self.user.id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_audioscrobbler_tsv_file(self.upload_file_path, self.user.id)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class ScaleCSVImport(BaseFileImportMixin):
@@ -297,9 +315,14 @@ class ScaleCSVImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = import_scale_csv(self.upload_file_path, self.user.id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_scale_csv(self.upload_file_path, self.user.id)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class TrailGPXImport(BaseFileImportMixin):
@@ -337,11 +360,16 @@ class TrailGPXImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = import_trail_gpx(
-            self.upload_file_path, self.user.id, self.original_filename
-        )
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_trail_gpx(
+                self.upload_file_path, self.user.id, self.original_filename
+            )
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class LastFmImport(BaseFileImportMixin):
@@ -391,11 +419,14 @@ class LastFmImport(BaseFileImportMixin):
                 last_processed = last_import.processed_finished
 
         self.mark_started()
-
-        scrobbles = lastfm.import_from_lastfm(last_processed, time_to=time_to)
-
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = lastfm.import_from_lastfm(last_processed, time_to=time_to)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class RetroarchImport(BaseFileImportMixin):
@@ -426,43 +457,48 @@ class RetroarchImport(BaseFileImportMixin):
             logger.info(f"You told me to force import from Retroarch")
 
         self.mark_started()
+        scrobbles = None
+        try:
+            if self.lrtl_file:
+                import os
+                import tempfile
+                import zipfile
 
-        if self.lrtl_file:
-            import os
-            import tempfile
-            import zipfile
+                tmpdir = tempfile.mkdtemp()
+                try:
+                    zip_path = os.path.join(tmpdir, "archive.zip")
+                    with open(zip_path, "wb") as f:
+                        f.write(self.lrtl_file.read())
+                    with zipfile.ZipFile(zip_path, "r") as zf:
+                        zf.extractall(tmpdir)
+                    os.unlink(zip_path)
+                    scrobbles = retroarch.import_retroarch_lrtl_files(
+                        tmpdir + "/",
+                        self.user.id,
+                    )
+                finally:
+                    import shutil
+
+                    shutil.rmtree(tmpdir, ignore_errors=True)
+            else:
+                if not self.user.profile.retroarch_path:
+                    logger.info(
+                        "Trying to import Retroarch logs, but user has no retroarch_path configured"
+                    )
+                    self.mark_finished()
+                    return
 
-            tmpdir = tempfile.mkdtemp()
-            try:
-                zip_path = os.path.join(tmpdir, "archive.zip")
-                with open(zip_path, "wb") as f:
-                    f.write(self.lrtl_file.read())
-                with zipfile.ZipFile(zip_path, "r") as zf:
-                    zf.extractall(tmpdir)
-                os.unlink(zip_path)
                 scrobbles = retroarch.import_retroarch_lrtl_files(
-                    tmpdir + "/",
+                    self.user.profile.retroarch_path,
                     self.user.id,
                 )
-            finally:
-                import shutil
 
-                shutil.rmtree(tmpdir, ignore_errors=True)
-        else:
-            if not self.user.profile.retroarch_path:
-                logger.info(
-                    "Tying to import Retroarch logs, but user has no retroarch_path configured"
-                )
-                self.mark_finished()
-                return
-
-            scrobbles = retroarch.import_retroarch_lrtl_files(
-                self.user.profile.retroarch_path,
-                self.user.id,
-            )
-
-        self.record_log(scrobbles)
-        self.mark_finished()
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class BGStatsImport(BaseFileImportMixin):
@@ -499,17 +535,21 @@ class BGStatsImport(BaseFileImportMixin):
             return
 
         self.mark_started()
+        try:
+            import json
 
-        import json
+            from scrobbles.scrobblers import email_scrobble_board_game
 
-        from scrobbles.scrobblers import email_scrobble_board_game
+            with open(self.upload_file_path, "r", encoding="utf-8") as f:
+                parsed_json = json.load(f)
+            scrobbles = email_scrobble_board_game(parsed_json, self.user_id)
 
-        with open(self.upload_file_path, "r", encoding="utf-8") as f:
-            parsed_json = json.load(f)
-        scrobbles = email_scrobble_board_game(parsed_json, self.user_id)
-
-        self.record_log(scrobbles)
-        self.mark_finished()
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class EBirdCSVImport(BaseFileImportMixin):
@@ -554,9 +594,16 @@ class EBirdCSVImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = import_birding_csv(self.upload_file_path, self.user_id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_birding_csv(
+                self.upload_file_path, self.user_id, record_error=self.record_error
+            )
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class ScrobbleQuerySet(models.QuerySet):