[tasks] Move retroarch importing to webdav

vrobbler/apps/scrobbles/importers/webdav.py

@@ -13,14 +13,11 @@ logger = logging.getLogger(__name__)
 
 DEFAULT_GPX_PATH = "var/gpx/"
 DEFAULT_KOREADER_PATH = "var/koreader/"
+DEFAULT_RETROARCH_PATH = "var/retroarch/"
 
 
 def import_from_webdav_for_all_users(restart=False):
-    """Grab a list of all users with WebDAV enabled and kickoff imports for them"""
+    """Iterate all WebDAV-enabled users, scanning each media-type directory."""
-
-    koreader_path = (
-        DEFAULT_KOREADER_PATH + "statistics.sqlite3"
-    )  # TODO Allow configuring this in user settings
     webdav_enabled_user_ids = UserProfile.objects.filter(
         webdav_url__isnull=False,
         webdav_user__isnull=False,
@@ -29,66 +26,87 @@ def import_from_webdav_for_all_users(restart=False):
     ).values_list("user_id", flat=True)
     logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
 
-    koreader_import_count = 0
+    ko_count = 0
-    trail_gpx_import_count = 0
+    gpx_count = 0
+    retro_count = 0
 
     for user_id in webdav_enabled_user_ids:
-        webdav_client = get_webdav_client(user_id)
+        client = get_webdav_client(user_id)
+        ko_count += scan_webdav_for_koreader(client, user_id, restart)
+        gpx_count += scan_webdav_for_gpx(client, user_id)
+        retro_count += scan_webdav_for_retroarch(client, user_id)
 
-        try:
+    logger.info(
-            webdav_client.info(koreader_path)
+        "WebDAV import complete",
-            koreader_found = True
+        extra={
-        except:
+            "koreader": ko_count,
-            koreader_found = False
+            "trail_gpx": gpx_count,
-            logger.info(
+            "retroarch": retro_count,
-                "No koreader stats file found on webdav",
+        },
-                extra={"user_id": user_id},
+    )
-            )
+    return ko_count, gpx_count, retro_count
 
-        trail_gpx_import_count += scan_webdav_for_gpx(webdav_client, user_id)
 
-        if koreader_found:
+def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
-            last_import = (
+    """Check for koreader statistics.sqlite3 and queue import if changed."""
-                KoReaderImport.objects.filter(
+    koreader_path = DEFAULT_KOREADER_PATH + "statistics.sqlite3"
-                    user_id=user_id, processed_finished__isnull=False
+    try:
-                )
+        webdav_client.info(koreader_path)
-                .order_by("processed_finished")
+    except:
-                .last()
+        logger.info(
-            )
+            "No koreader stats file found on webdav",
+            extra={"user_id": user_id},
+        )
+        return 0
 
-            new_hash = get_file_md5_hash(fetch_file_from_webdav(1))
+    last_import = (
-            old_hash = None
+        KoReaderImport.objects.filter(
-            if last_import:
+            user_id=user_id, processed_finished__isnull=False
-                old_hash = last_import.file_md5_hash()
+        )
+        .order_by("processed_finished")
+        .last()
+    )
 
-            if old_hash and new_hash == old_hash:
+    file_path = fetch_file_from_webdav(user_id)
-                logger.info(
+    if not file_path:
-                    "koreader stats file has not changed",
+        logger.warning(
-                    extra={
+            "Could not fetch koreader file from webdav",
-                        "user_id": user_id,
+            extra={"user_id": user_id},
-                        "new_hash": new_hash,
+        )
-                        "old_hash": old_hash,
+        return 0
-                        "last_import_id": last_import.id,
-                    },
-                )
-                continue
 
-            koreader_import, created = KoReaderImport.objects.get_or_create(
+    new_hash = get_file_md5_hash(file_path)
-                user_id=user_id, processed_finished__isnull=True
+    old_hash = last_import.file_md5_hash() if last_import else None
-            )
 
-            if not created and not restart:
+    if last_import and new_hash == old_hash:
-                logger.info(
+        logger.info(
-                    f"Not resuming failed KoReader import {koreader_import.id} for user {user_id}, use restart=True to restart"
+            "koreader stats file has not changed",
-                )
+            extra={
-                continue
+                "user_id": user_id,
+                "new_hash": new_hash,
+                "old_hash": old_hash,
+                "last_import_id": last_import.id,
+            },
+        )
+        os.unlink(file_path)
+        return 0
 
-            koreader_import.save_sqlite_file_to_self(koreader_file_path)
+    koreader_import, created = KoReaderImport.objects.get_or_create(
+        user_id=user_id, processed_finished__isnull=True
+    )
 
-            process_koreader_import.delay(koreader_import.id)
+    if not created and not restart:
-            koreader_import_count += 1
+        logger.info(
+            f"Not resuming failed KoReader import {koreader_import.id} "
+            f"for user {user_id}, use restart=True to restart"
+        )
+        os.unlink(file_path)
+        return 0
 
-    return koreader_import_count, trail_gpx_import_count
+    koreader_import.save_sqlite_file_to_self(file_path)
+    process_koreader_import.delay(koreader_import.id)
+    os.unlink(file_path)
+    return 1
 
 
 def scan_webdav_for_gpx(webdav_client, user_id):
@@ -142,3 +160,72 @@ def scan_webdav_for_gpx(webdav_client, user_id):
             os.unlink(tmp.name)
 
     return new_imports
+
+
+def scan_webdav_for_retroarch(webdav_client, user_id):
+    """Download .lrtl files from WebDAV and import retroarch playlog data."""
+    retroarch_path = DEFAULT_RETROARCH_PATH
+    try:
+        webdav_client.info(retroarch_path)
+    except:
+        logger.info(
+            "No var/retroarch/ directory on webdav", extra={"user_id": user_id}
+        )
+        return 0
+
+    try:
+        files = webdav_client.list(retroarch_path)
+    except Exception as e:
+        logger.warning(
+            "Could not list var/retroarch/",
+            extra={"user_id": user_id, "error": str(e)},
+        )
+        return 0
+
+    lrtl_files = [
+        fname
+        for fname in files
+        if os.path.basename(fname).lower().endswith(".lrtl")
+    ]
+    if not lrtl_files:
+        logger.info(
+            "No .lrtl files found on webdav", extra={"user_id": user_id}
+        )
+        return 0
+
+    from scrobbles.models import RetroarchImport
+    from scrobbles.tasks import process_retroarch_import
+
+    already_imported = set(
+        RetroarchImport.objects.filter(user_id=user_id).values_list(
+            "original_filename", flat=True
+        )
+    )
+
+    new_imports = 0
+    for fname in lrtl_files:
+        basename = os.path.basename(fname)
+        if basename in already_imported:
+            logger.debug(f"Skipping already-imported {basename}")
+            continue
+
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=basename)
+        try:
+            webdav_client.download_sync(
+                remote_path=f"{retroarch_path}/{basename}",
+                local_path=tmp.name,
+            )
+            imp = RetroarchImport.objects.create(
+                user_id=user_id,
+                original_filename=basename,
+            )
+            with open(tmp.name, "rb") as f:
+                imp.lrtl_file.save(basename, f, save=True)
+            process_retroarch_import.delay(imp.id)
+            new_imports += 1
+        except Exception as e:
+            logger.error(f"Failed to import retroarch {basename}: {e}")
+        finally:
+            os.unlink(tmp.name)
+
+    return new_imports

vrobbler/apps/scrobbles/management/commands/import_from_webdav.py

@@ -14,5 +14,10 @@ class Command(BaseCommand):
         restart = False
         if options["restart"]:
             restart = True
-        ko_count, gpx_count = webdav.import_from_webdav_for_all_users(restart=restart)
+        ko_count, gpx_count, retro_count = webdav.import_from_webdav_for_all_users(
-        print(f"Started {ko_count} KOReader and {gpx_count} Trail GPX WebDAV imports")
+            restart=restart
+        )
+        print(
+            f"Started {ko_count} KOReader, {gpx_count} Trail GPX, "
+            f"and {retro_count} Retroarch WebDAV imports"
+        )

vrobbler/apps/scrobbles/migrations/0079_retroarchimport_lrtl_file.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.29 on 2026-05-22 15:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0078_trailgpximport"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="retroarchimport",
+            name="lrtl_file",
+            field=models.FileField(
+                blank=True, null=True, upload_to="scrobbles/lrtl_file/"
+            ),
+        ),
+        migrations.AddField(
+            model_name="retroarchimport",
+            name="original_filename",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -399,6 +399,9 @@ class RetroarchImport(BaseFileImportMixin):
     def get_absolute_url(self):
         return reverse("scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid})
 
+    original_filename = models.CharField(max_length=255, **BNULL)
+    lrtl_file = models.FileField(upload_to="scrobbles/lrtl_file/", **BNULL)
+
     def process(self, import_all=False, force=False):
         """Import scrobbles found on Retroarch"""
         if self.user.id == 1:
@@ -411,17 +414,39 @@ class RetroarchImport(BaseFileImportMixin):
         if force:
             logger.info(f"You told me to force import from Retroarch")
 
-        if not self.user.profile.retroarch_path:
-            logger.info(
-                "Tying to import Retroarch logs, but user has no retroarch_path configured"
-            )
-
         self.mark_started()
 
-        scrobbles = retroarch.import_retroarch_lrtl_files(
+        if self.lrtl_file:
-            self.user.profile.retroarch_path,
+            import tempfile
-            self.user.id,
+            import os
-        )
+
+            tmpdir = tempfile.mkdtemp()
+            try:
+                src_path = self.lrtl_file.path
+                basename = self.original_filename or os.path.basename(src_path)
+                dst = os.path.join(tmpdir, basename)
+                with open(dst, "wb") as f:
+                    f.write(self.lrtl_file.read())
+                scrobbles = retroarch.import_retroarch_lrtl_files(
+                    tmpdir + "/",
+                    self.user.id,
+                )
+            finally:
+                import shutil
+
+                shutil.rmtree(tmpdir, ignore_errors=True)
+        else:
+            if not self.user.profile.retroarch_path:
+                logger.info(
+                    "Tying to import Retroarch logs, but user has no retroarch_path configured"
+                )
+                self.mark_finished()
+                return
+
+            scrobbles = retroarch.import_retroarch_lrtl_files(
+                self.user.profile.retroarch_path,
+                self.user.id,
+            )
 
         self.record_log(scrobbles)
         self.mark_finished()

vrobbler/apps/scrobbles/tasks.py

@@ -387,7 +387,19 @@ def import_from_lastfm_all_users():
 
 @shared_task
 def import_from_retroarch_all_users():
-    """Import RetroArch scrobbles for all users (replaces @daily cron)."""
+    """Import RetroArch scrobbles for all users (replaces @daily cron).
+
+    Deprecated: retroarch .lrtl files are now picked up by the WebDAV
+    importer (scan_webdav_for_retroarch). This task remains for manual use.
+    """
+    import warnings
+
+    warnings.warn(
+        "import_from_retroarch_all_users is deprecated. "
+        "Upload .lrtl files to WebDAV var/retroarch/ instead.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     from scrobbles.utils import import_retroarch_for_all_users
 
     import_retroarch_for_all_users()

vrobbler/settings.py

@@ -135,10 +135,6 @@ CELERY_BEAT_SCHEDULE = {
         "task": "scrobbles.tasks.import_from_lastfm_all_users",
         "schedule": crontab(minute="*/30"),
     },
-    "import-from-retroarch": {
-        "task": "scrobbles.tasks.import_from_retroarch_all_users",
-        "schedule": crontab(hour=0, minute=0),
-    },
     "import-from-webdav": {
         "task": "scrobbles.tasks.import_from_webdav_all_users",
         "schedule": crontab(minute="*/2"),