[importers] Add error_log to surface import errors

vrobbler/apps/scrobbles/models.py

@@ -74,6 +74,7 @@ class BaseFileImportMixin(TimeStampedModel):
     processed_finished = models.DateTimeField(**BNULL)
     process_log = models.TextField(**BNULL)
     process_count = models.IntegerField(**BNULL)
+    error_log = models.TextField(**BNULL)
 
     class Meta:
         abstract = True
@@ -158,6 +159,14 @@ class BaseFileImportMixin(TimeStampedModel):
         self.process_count = len(scrobbles)
         self.save(update_fields=["process_log", "process_count"])
 
+    def record_error(self, error_message):
+        log_line = f"{timezone.now().isoformat()}: {error_message}"
+        if self.error_log:
+            self.error_log += "\n" + log_line
+        else:
+            self.error_log = log_line
+        self.save(update_fields=["error_log"])
+
     @property
     def upload_file_path(self):
         raise NotImplementedError
@@ -213,9 +222,14 @@ class KoReaderImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = process_koreader_sqlite_file(self.upload_file_path, self.user.id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = process_koreader_sqlite_file(self.upload_file_path, self.user.id)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class AudioScrobblerTSVImport(BaseFileImportMixin):
@@ -255,10 +269,14 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-
-        scrobbles = import_audioscrobbler_tsv_file(self.upload_file_path, self.user.id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_audioscrobbler_tsv_file(self.upload_file_path, self.user.id)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class ScaleCSVImport(BaseFileImportMixin):
@@ -297,9 +315,14 @@ class ScaleCSVImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = import_scale_csv(self.upload_file_path, self.user.id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_scale_csv(self.upload_file_path, self.user.id)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class TrailGPXImport(BaseFileImportMixin):
@@ -337,11 +360,16 @@ class TrailGPXImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = import_trail_gpx(
-            self.upload_file_path, self.user.id, self.original_filename
-        )
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_trail_gpx(
+                self.upload_file_path, self.user.id, self.original_filename
+            )
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class LastFmImport(BaseFileImportMixin):
@@ -391,11 +419,14 @@ class LastFmImport(BaseFileImportMixin):
                 last_processed = last_import.processed_finished
 
         self.mark_started()
-
-        scrobbles = lastfm.import_from_lastfm(last_processed, time_to=time_to)
-
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = lastfm.import_from_lastfm(last_processed, time_to=time_to)
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class RetroarchImport(BaseFileImportMixin):
@@ -426,43 +457,48 @@ class RetroarchImport(BaseFileImportMixin):
             logger.info(f"You told me to force import from Retroarch")
 
         self.mark_started()
+        scrobbles = None
+        try:
+            if self.lrtl_file:
+                import os
+                import tempfile
+                import zipfile
 
-        if self.lrtl_file:
-            import os
-            import tempfile
-            import zipfile
+                tmpdir = tempfile.mkdtemp()
+                try:
+                    zip_path = os.path.join(tmpdir, "archive.zip")
+                    with open(zip_path, "wb") as f:
+                        f.write(self.lrtl_file.read())
+                    with zipfile.ZipFile(zip_path, "r") as zf:
+                        zf.extractall(tmpdir)
+                    os.unlink(zip_path)
+                    scrobbles = retroarch.import_retroarch_lrtl_files(
+                        tmpdir + "/",
+                        self.user.id,
+                    )
+                finally:
+                    import shutil
+
+                    shutil.rmtree(tmpdir, ignore_errors=True)
+            else:
+                if not self.user.profile.retroarch_path:
+                    logger.info(
+                        "Trying to import Retroarch logs, but user has no retroarch_path configured"
+                    )
+                    self.mark_finished()
+                    return
 
-            tmpdir = tempfile.mkdtemp()
-            try:
-                zip_path = os.path.join(tmpdir, "archive.zip")
-                with open(zip_path, "wb") as f:
-                    f.write(self.lrtl_file.read())
-                with zipfile.ZipFile(zip_path, "r") as zf:
-                    zf.extractall(tmpdir)
-                os.unlink(zip_path)
                 scrobbles = retroarch.import_retroarch_lrtl_files(
-                    tmpdir + "/",
+                    self.user.profile.retroarch_path,
                     self.user.id,
                 )
-            finally:
-                import shutil
 
-                shutil.rmtree(tmpdir, ignore_errors=True)
-        else:
-            if not self.user.profile.retroarch_path:
-                logger.info(
-                    "Tying to import Retroarch logs, but user has no retroarch_path configured"
-                )
-                self.mark_finished()
-                return
-
-            scrobbles = retroarch.import_retroarch_lrtl_files(
-                self.user.profile.retroarch_path,
-                self.user.id,
-            )
-
-        self.record_log(scrobbles)
-        self.mark_finished()
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class BGStatsImport(BaseFileImportMixin):
@@ -499,17 +535,21 @@ class BGStatsImport(BaseFileImportMixin):
             return
 
         self.mark_started()
+        try:
+            import json
 
-        import json
+            from scrobbles.scrobblers import email_scrobble_board_game
 
-        from scrobbles.scrobblers import email_scrobble_board_game
+            with open(self.upload_file_path, "r", encoding="utf-8") as f:
+                parsed_json = json.load(f)
+            scrobbles = email_scrobble_board_game(parsed_json, self.user_id)
 
-        with open(self.upload_file_path, "r", encoding="utf-8") as f:
-            parsed_json = json.load(f)
-        scrobbles = email_scrobble_board_game(parsed_json, self.user_id)
-
-        self.record_log(scrobbles)
-        self.mark_finished()
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class EBirdCSVImport(BaseFileImportMixin):
@@ -554,9 +594,16 @@ class EBirdCSVImport(BaseFileImportMixin):
             return
 
         self.mark_started()
-        scrobbles = import_birding_csv(self.upload_file_path, self.user_id)
-        self.record_log(scrobbles)
-        self.mark_finished()
+        try:
+            scrobbles = import_birding_csv(
+                self.upload_file_path, self.user_id, record_error=self.record_error
+            )
+            self.record_log(scrobbles)
+        except Exception as e:
+            self.record_error(f"Import failed: {e}")
+            logger.exception(f"Import failed for {self}")
+        finally:
+            self.mark_finished()
 
 
 class ScrobbleQuerySet(models.QuerySet):