[importers] Retroarch webdav imports oldest first

vrobbler/apps/scrobbles/importers/webdav.py

@@ -63,7 +63,10 @@ def import_from_webdav_for_all_users(
         )
         logger.info("Scanning WebDAV retroarch for user %s", user_id)
         retro_count += scan_webdav_for_retroarch(
-            client, user_id, update_hash_only=update_retroarch_hash
+            client,
+            user_id,
+            update_hash_only=update_retroarch_hash,
+            include_processed=include_processed,
         )
         logger.info("Scanning WebDAV bgstats for user %s", user_id)
         bgstats_count += scan_webdav_for_bgstats(
@@ -312,7 +315,9 @@ def scan_webdav_for_gpx(webdav_client, user_id, include_processed=False):
     return new_imports
 
 
-def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
+def scan_webdav_for_retroarch(
+    webdav_client, user_id, update_hash_only=False, include_processed=False
+):
     """Check for new .lrtl files on WebDAV, download+import if changed.
 
     Uses ETags from a single PROPFIND to detect changes without downloading
@@ -321,9 +326,16 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
     When *update_hash_only* is True, mismatched hashes update the last import's
     stored hash without re-downloading — useful when migrating to a new hash
     scheme to avoid a one-time re-import.
+
+    When *include_processed* is True, also imports files from the processed/
+    subdirectory.  These are sorted chronologically by their YYYYMMDDHHMM-
+    timestamp prefix so that the importer always sees the earliest snapshot
+    first, which is critical for correct incremental scrobbling since
+    retroarch overwrites the same .lrtl filename.
     """
     import hashlib
     import shutil
+    import re
     import zipfile
     from datetime import datetime
 
@@ -343,8 +355,9 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
     except Exception:
         pass
 
+    # Collect files from root directory
     try:
-        files = webdav_client.list(retroarch_path, get_info=True)
+        root_files = webdav_client.list(retroarch_path, get_info=True)
     except Exception as e:
         logger.warning(
             "Could not list var/retroarch/",
@@ -353,18 +366,50 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
         return 0
 
     # Extract basename from path since the webdav adapter returns name=None
-    for f in files:
+    for f in root_files:
         if f.get("path") and not f.get("name"):
             f["name"] = os.path.basename(f["path"])
 
-    lrtl_files = sorted(
+    lrtl_files = [
-        [f for f in files if (f.get("name") or "").lower().endswith(".lrtl")],
+        f for f in root_files
-        key=lambda x: (x.get("name") or ""),
+        if (f.get("name") or "").lower().endswith(".lrtl")
-    )
+    ]
+
+    # Optionally include historical files from processed/
+    if include_processed:
+        try:
+            processed_files = webdav_client.list(processed_dir, get_info=True)
+        except Exception as e:
+            logger.warning(
+                "Could not list var/retroarch/processed/",
+                extra={"user_id": user_id, "error": str(e)},
+            )
+        else:
+            for f in processed_files:
+                if f.get("path") and not f.get("name"):
+                    f["name"] = os.path.basename(f["path"])
+            lrtl_files.extend([
+                f for f in processed_files
+                if (f.get("name") or "").lower().endswith(".lrtl")
+            ])
+
     if not lrtl_files:
         logger.info("No .lrtl files found on webdav", extra={"user_id": user_id})
         return 0
 
+    # Sort chronologically: processed files (timestamp prefixed) first,
+    # current root files last.
+    ts_pattern = re.compile(r"^(\d{12})-")
+    def sort_key(f):
+        name = f.get("name") or ""
+        m = ts_pattern.match(name)
+        if m:
+            return m.group(1)
+        # Root files (no timestamp) go after all historical snapshots
+        return "999999999999"
+
+    lrtl_files.sort(key=sort_key)
+
     # Don't queue if one is still being processed
     if RetroarchImport.objects.filter(
         user_id=user_id, processed_finished__isnull=True
@@ -382,7 +427,6 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
         hasher.update((f.get("etag") or f.get("modified") or "").encode())
     content_hash = hasher.hexdigest()
 
-    # Skip if the last completed import already has this hash
     last_import = (
         RetroarchImport.objects.filter(
             user_id=user_id, processed_finished__isnull=False
@@ -390,7 +434,9 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
         .order_by("-processed_finished")
         .first()
     )
-    if last_import and last_import.files_hash == content_hash:
+
+    # Skip unchanged root files only, never skip when re-processing history
+    if not include_processed and last_import and last_import.files_hash == content_hash:
         logger.info(
             "Retroarch lrtl files unchanged for user, skipping",
             extra={"user_id": user_id},
@@ -414,7 +460,11 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
         imported_filenames = []
         for f in lrtl_files:
             basename = f.get("name")
-            remote_path = f"{retroarch_path}{basename}"
+            # Determine source directory: processed files already live under processed/
+            is_processed = ts_pattern.match(basename)
+            source_dir = processed_dir if is_processed else retroarch_path
+
+            remote_path = f"{source_dir}{basename}"
             dst = os.path.join(download_dir, basename)
             try:
                 webdav_client.download_sync(
@@ -423,13 +473,16 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False):
                 )
                 downloaded.append(basename)
 
-                ts = datetime.now().strftime("%Y%m%d%H%M%S")
+                if is_processed:
-                processed_name = f"{ts}-{basename}"
+                    imported_filenames.append(basename)
-                imported_filenames.append(processed_name)
+                else:
-                webdav_client.move(
+                    ts = datetime.now().strftime("%Y%m%d%H%M%S")
-                    remote_path,
+                    processed_name = f"{ts}-{basename}"
-                    f"{processed_dir}{processed_name}",
+                    imported_filenames.append(processed_name)
-                )
+                    webdav_client.move(
+                        remote_path,
+                        f"{processed_dir}{processed_name}",
+                    )
             except Exception as e:
                 logger.error("Failed to download retroarch %s: %s", basename, e)