diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index 917e83b..1aef8ba 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -163,7 +163,13 @@ def scan_webdav_for_gpx(webdav_client, user_id): def scan_webdav_for_retroarch(webdav_client, user_id): - """Download .lrtl files from WebDAV and import retroarch playlog data.""" + """Download all .lrtl files from WebDAV, zip them, queue one import.""" + import zipfile + from datetime import datetime + + from scrobbles.models import RetroarchImport + from scrobbles.tasks import process_retroarch_import + retroarch_path = DEFAULT_RETROARCH_PATH try: webdav_client.info(retroarch_path) @@ -182,50 +188,67 @@ def scan_webdav_for_retroarch(webdav_client, user_id): ) return 0 - lrtl_files = [ - fname + lrtl_basenames = sorted( + os.path.basename(fname) for fname in files - if os.path.basename(fname).lower().endswith(".lrtl") - ] - if not lrtl_files: + if fname.lower().endswith(".lrtl") + ) + if not lrtl_basenames: + logger.info("No .lrtl files found on webdav", extra={"user_id": user_id}) + return 0 + + # Don't queue if one is still being processed + if RetroarchImport.objects.filter( + user_id=user_id, processed_finished__isnull=True + ).exists(): logger.info( - "No .lrtl files found on webdav", extra={"user_id": user_id} + "Retroarch import already pending for user", + extra={"user_id": user_id}, ) return 0 - from scrobbles.models import RetroarchImport - from scrobbles.tasks import process_retroarch_import + download_dir = tempfile.mkdtemp() + try: + downloaded = [] + for basename in lrtl_basenames: + dst = os.path.join(download_dir, basename) + try: + webdav_client.download_sync( + remote_path=f"{retroarch_path}/{basename}", + local_path=dst, + ) + downloaded.append(basename) + except Exception as e: + logger.error( + "Failed to download retroarch %s: %s", basename, e + ) - already_imported = set( - RetroarchImport.objects.filter(user_id=user_id).values_list( - "original_filename", flat=True + if not downloaded: + return 0 + + zip_path = os.path.join( + download_dir, f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip" ) - ) + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: + for basename in downloaded: + zf.write(os.path.join(download_dir, basename), basename) - new_imports = 0 - for fname in lrtl_files: - basename = os.path.basename(fname) - if basename in already_imported: - logger.debug(f"Skipping already-imported {basename}") - continue - - tmp = tempfile.NamedTemporaryFile(delete=False, suffix=basename) - try: - webdav_client.download_sync( - remote_path=f"{retroarch_path}/{basename}", - local_path=tmp.name, + imp = RetroarchImport.objects.create( + user_id=user_id, + original_filename=f"batch-{len(downloaded)}-files", + ) + with open(zip_path, "rb") as f: + imp.lrtl_file.save( + f"retroarch-batch-{imp.uuid}.zip", f, save=True ) - imp = RetroarchImport.objects.create( - user_id=user_id, - original_filename=basename, - ) - with open(tmp.name, "rb") as f: - imp.lrtl_file.save(basename, f, save=True) - process_retroarch_import.delay(imp.id) - new_imports += 1 - except Exception as e: - logger.error(f"Failed to import retroarch {basename}: {e}") - finally: - os.unlink(tmp.name) + process_retroarch_import.delay(imp.id) + logger.info( + "Queued retroarch import %s with %d file(s)", + imp.uuid, + len(downloaded), + ) + return 1 + finally: + import shutil - return new_imports + shutil.rmtree(download_dir, ignore_errors=True) diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index e7c21c7..5e09cd1 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -419,14 +419,16 @@ class RetroarchImport(BaseFileImportMixin): if self.lrtl_file: import tempfile import os + import zipfile tmpdir = tempfile.mkdtemp() try: - src_path = self.lrtl_file.path - basename = self.original_filename or os.path.basename(src_path) - dst = os.path.join(tmpdir, basename) - with open(dst, "wb") as f: + zip_path = os.path.join(tmpdir, "archive.zip") + with open(zip_path, "wb") as f: f.write(self.lrtl_file.read()) + with zipfile.ZipFile(zip_path, "r") as zf: + zf.extractall(tmpdir) + os.unlink(zip_path) scrobbles = retroarch.import_retroarch_lrtl_files( tmpdir + "/", self.user.id,