[tasks] Move retroarch importing to webdav

vrobbler/apps/scrobbles/importers/webdav.py

@@ -13,14 +13,11 @@ logger = logging.getLogger(__name__)
 
 DEFAULT_GPX_PATH = "var/gpx/"
 DEFAULT_KOREADER_PATH = "var/koreader/"
+DEFAULT_RETROARCH_PATH = "var/retroarch/"
 
 
 def import_from_webdav_for_all_users(restart=False):
-    """Grab a list of all users with WebDAV enabled and kickoff imports for them"""
-
-    koreader_path = (
-        DEFAULT_KOREADER_PATH + "statistics.sqlite3"
-    )  # TODO Allow configuring this in user settings
+    """Iterate all WebDAV-enabled users, scanning each media-type directory."""
     webdav_enabled_user_ids = UserProfile.objects.filter(
         webdav_url__isnull=False,
         webdav_user__isnull=False,
@@ -29,66 +26,87 @@ def import_from_webdav_for_all_users(restart=False):
     ).values_list("user_id", flat=True)
     logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
 
-    koreader_import_count = 0
-    trail_gpx_import_count = 0
+    ko_count = 0
+    gpx_count = 0
+    retro_count = 0
 
     for user_id in webdav_enabled_user_ids:
-        webdav_client = get_webdav_client(user_id)
+        client = get_webdav_client(user_id)
+        ko_count += scan_webdav_for_koreader(client, user_id, restart)
+        gpx_count += scan_webdav_for_gpx(client, user_id)
+        retro_count += scan_webdav_for_retroarch(client, user_id)
 
-        try:
-            webdav_client.info(koreader_path)
-            koreader_found = True
-        except:
-            koreader_found = False
-            logger.info(
-                "No koreader stats file found on webdav",
-                extra={"user_id": user_id},
-            )
+    logger.info(
+        "WebDAV import complete",
+        extra={
+            "koreader": ko_count,
+            "trail_gpx": gpx_count,
+            "retroarch": retro_count,
+        },
+    )
+    return ko_count, gpx_count, retro_count
 
-        trail_gpx_import_count += scan_webdav_for_gpx(webdav_client, user_id)
 
-        if koreader_found:
-            last_import = (
-                KoReaderImport.objects.filter(
-                    user_id=user_id, processed_finished__isnull=False
-                )
-                .order_by("processed_finished")
-                .last()
-            )
+def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
+    """Check for koreader statistics.sqlite3 and queue import if changed."""
+    koreader_path = DEFAULT_KOREADER_PATH + "statistics.sqlite3"
+    try:
+        webdav_client.info(koreader_path)
+    except:
+        logger.info(
+            "No koreader stats file found on webdav",
+            extra={"user_id": user_id},
+        )
+        return 0
 
-            new_hash = get_file_md5_hash(fetch_file_from_webdav(1))
-            old_hash = None
-            if last_import:
-                old_hash = last_import.file_md5_hash()
+    last_import = (
+        KoReaderImport.objects.filter(
+            user_id=user_id, processed_finished__isnull=False
+        )
+        .order_by("processed_finished")
+        .last()
+    )
 
-            if old_hash and new_hash == old_hash:
-                logger.info(
-                    "koreader stats file has not changed",
-                    extra={
-                        "user_id": user_id,
-                        "new_hash": new_hash,
-                        "old_hash": old_hash,
-                        "last_import_id": last_import.id,
-                    },
-                )
-                continue
+    file_path = fetch_file_from_webdav(user_id)
+    if not file_path:
+        logger.warning(
+            "Could not fetch koreader file from webdav",
+            extra={"user_id": user_id},
+        )
+        return 0
 
-            koreader_import, created = KoReaderImport.objects.get_or_create(
-                user_id=user_id, processed_finished__isnull=True
-            )
+    new_hash = get_file_md5_hash(file_path)
+    old_hash = last_import.file_md5_hash() if last_import else None
 
-            if not created and not restart:
-                logger.info(
-                    f"Not resuming failed KoReader import {koreader_import.id} for user {user_id}, use restart=True to restart"
-                )
-                continue
+    if last_import and new_hash == old_hash:
+        logger.info(
+            "koreader stats file has not changed",
+            extra={
+                "user_id": user_id,
+                "new_hash": new_hash,
+                "old_hash": old_hash,
+                "last_import_id": last_import.id,
+            },
+        )
+        os.unlink(file_path)
+        return 0
 
-            koreader_import.save_sqlite_file_to_self(koreader_file_path)
+    koreader_import, created = KoReaderImport.objects.get_or_create(
+        user_id=user_id, processed_finished__isnull=True
+    )
 
-            process_koreader_import.delay(koreader_import.id)
-            koreader_import_count += 1
+    if not created and not restart:
+        logger.info(
+            f"Not resuming failed KoReader import {koreader_import.id} "
+            f"for user {user_id}, use restart=True to restart"
+        )
+        os.unlink(file_path)
+        return 0
 
-    return koreader_import_count, trail_gpx_import_count
+    koreader_import.save_sqlite_file_to_self(file_path)
+    process_koreader_import.delay(koreader_import.id)
+    os.unlink(file_path)
+    return 1
 
 
 def scan_webdav_for_gpx(webdav_client, user_id):
@@ -142,3 +160,72 @@ def scan_webdav_for_gpx(webdav_client, user_id):
             os.unlink(tmp.name)
 
     return new_imports
+
+
+def scan_webdav_for_retroarch(webdav_client, user_id):
+    """Download .lrtl files from WebDAV and import retroarch playlog data."""
+    retroarch_path = DEFAULT_RETROARCH_PATH
+    try:
+        webdav_client.info(retroarch_path)
+    except:
+        logger.info(
+            "No var/retroarch/ directory on webdav", extra={"user_id": user_id}
+        )
+        return 0
+
+    try:
+        files = webdav_client.list(retroarch_path)
+    except Exception as e:
+        logger.warning(
+            "Could not list var/retroarch/",
+            extra={"user_id": user_id, "error": str(e)},
+        )
+        return 0
+
+    lrtl_files = [
+        fname
+        for fname in files
+        if os.path.basename(fname).lower().endswith(".lrtl")
+    ]
+    if not lrtl_files:
+        logger.info(
+            "No .lrtl files found on webdav", extra={"user_id": user_id}
+        )
+        return 0
+
+    from scrobbles.models import RetroarchImport
+    from scrobbles.tasks import process_retroarch_import
+
+    already_imported = set(
+        RetroarchImport.objects.filter(user_id=user_id).values_list(
+            "original_filename", flat=True
+        )
+    )
+
+    new_imports = 0
+    for fname in lrtl_files:
+        basename = os.path.basename(fname)
+        if basename in already_imported:
+            logger.debug(f"Skipping already-imported {basename}")
+            continue
+
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=basename)
+        try:
+            webdav_client.download_sync(
+                remote_path=f"{retroarch_path}/{basename}",
+                local_path=tmp.name,
+            )
+            imp = RetroarchImport.objects.create(
+                user_id=user_id,
+                original_filename=basename,
+            )
+            with open(tmp.name, "rb") as f:
+                imp.lrtl_file.save(basename, f, save=True)
+            process_retroarch_import.delay(imp.id)
+            new_imports += 1
+        except Exception as e:
+            logger.error(f"Failed to import retroarch {basename}: {e}")
+        finally:
+            os.unlink(tmp.name)
+
+    return new_imports