[videogames] Save lrtl files in zip file on import

vrobbler/apps/scrobbles/importers/webdav.py

@@ -163,7 +163,13 @@ def scan_webdav_for_gpx(webdav_client, user_id):
 
 
 def scan_webdav_for_retroarch(webdav_client, user_id):
-    """Download .lrtl files from WebDAV and import retroarch playlog data."""
+    """Download all .lrtl files from WebDAV, zip them, queue one import."""
+    import zipfile
+    from datetime import datetime
+
+    from scrobbles.models import RetroarchImport
+    from scrobbles.tasks import process_retroarch_import
+
     retroarch_path = DEFAULT_RETROARCH_PATH
     try:
         webdav_client.info(retroarch_path)
@@ -182,50 +188,67 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
         )
         return 0
 
-    lrtl_files = [
+    lrtl_basenames = sorted(
-        fname
+        os.path.basename(fname)
         for fname in files
-        if os.path.basename(fname).lower().endswith(".lrtl")
+        if fname.lower().endswith(".lrtl")
-    ]
+    )
-    if not lrtl_files:
+    if not lrtl_basenames:
+        logger.info("No .lrtl files found on webdav", extra={"user_id": user_id})
+        return 0
+
+    # Don't queue if one is still being processed
+    if RetroarchImport.objects.filter(
+        user_id=user_id, processed_finished__isnull=True
+    ).exists():
         logger.info(
-            "No .lrtl files found on webdav", extra={"user_id": user_id}
+            "Retroarch import already pending for user",
+            extra={"user_id": user_id},
         )
         return 0
 
-    from scrobbles.models import RetroarchImport
+    download_dir = tempfile.mkdtemp()
-    from scrobbles.tasks import process_retroarch_import
+    try:
+        downloaded = []
+        for basename in lrtl_basenames:
+            dst = os.path.join(download_dir, basename)
+            try:
+                webdav_client.download_sync(
+                    remote_path=f"{retroarch_path}/{basename}",
+                    local_path=dst,
+                )
+                downloaded.append(basename)
+            except Exception as e:
+                logger.error(
+                    "Failed to download retroarch %s: %s", basename, e
+                )
 
-    already_imported = set(
+        if not downloaded:
-        RetroarchImport.objects.filter(user_id=user_id).values_list(
+            return 0
-            "original_filename", flat=True
+
+        zip_path = os.path.join(
+            download_dir, f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip"
         )
-    )
+        with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
+            for basename in downloaded:
+                zf.write(os.path.join(download_dir, basename), basename)
 
-    new_imports = 0
+        imp = RetroarchImport.objects.create(
-    for fname in lrtl_files:
+            user_id=user_id,
-        basename = os.path.basename(fname)
+            original_filename=f"batch-{len(downloaded)}-files",
-        if basename in already_imported:
+        )
-            logger.debug(f"Skipping already-imported {basename}")
+        with open(zip_path, "rb") as f:
-            continue
+            imp.lrtl_file.save(
-
+                f"retroarch-batch-{imp.uuid}.zip", f, save=True
-        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=basename)
-        try:
-            webdav_client.download_sync(
-                remote_path=f"{retroarch_path}/{basename}",
-                local_path=tmp.name,
             )
-            imp = RetroarchImport.objects.create(
+        process_retroarch_import.delay(imp.id)
-                user_id=user_id,
+        logger.info(
-                original_filename=basename,
+            "Queued retroarch import %s with %d file(s)",
-            )
+            imp.uuid,
-            with open(tmp.name, "rb") as f:
+            len(downloaded),
-                imp.lrtl_file.save(basename, f, save=True)
+        )
-            process_retroarch_import.delay(imp.id)
+        return 1
-            new_imports += 1
+    finally:
-        except Exception as e:
+        import shutil
-            logger.error(f"Failed to import retroarch {basename}: {e}")
-        finally:
-            os.unlink(tmp.name)
 
-    return new_imports
+        shutil.rmtree(download_dir, ignore_errors=True)

vrobbler/apps/scrobbles/models.py

@@ -419,14 +419,16 @@ class RetroarchImport(BaseFileImportMixin):
         if self.lrtl_file:
             import tempfile
             import os
+            import zipfile
 
             tmpdir = tempfile.mkdtemp()
             try:
-                src_path = self.lrtl_file.path
+                zip_path = os.path.join(tmpdir, "archive.zip")
-                basename = self.original_filename or os.path.basename(src_path)
+                with open(zip_path, "wb") as f:
-                dst = os.path.join(tmpdir, basename)
-                with open(dst, "wb") as f:
                     f.write(self.lrtl_file.read())
+                with zipfile.ZipFile(zip_path, "r") as zf:
+                    zf.extractall(tmpdir)
+                os.unlink(zip_path)
                 scrobbles = retroarch.import_retroarch_lrtl_files(
                     tmpdir + "/",
                     self.user.id,