diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index 8752878..e9f3cb3 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -63,7 +63,10 @@ def import_from_webdav_for_all_users( ) logger.info("Scanning WebDAV retroarch for user %s", user_id) retro_count += scan_webdav_for_retroarch( - client, user_id, update_hash_only=update_retroarch_hash + client, + user_id, + update_hash_only=update_retroarch_hash, + include_processed=include_processed, ) logger.info("Scanning WebDAV bgstats for user %s", user_id) bgstats_count += scan_webdav_for_bgstats( @@ -312,7 +315,9 @@ def scan_webdav_for_gpx(webdav_client, user_id, include_processed=False): return new_imports -def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): +def scan_webdav_for_retroarch( + webdav_client, user_id, update_hash_only=False, include_processed=False +): """Check for new .lrtl files on WebDAV, download+import if changed. Uses ETags from a single PROPFIND to detect changes without downloading @@ -321,9 +326,16 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): When *update_hash_only* is True, mismatched hashes update the last import's stored hash without re-downloading — useful when migrating to a new hash scheme to avoid a one-time re-import. + + When *include_processed* is True, also imports files from the processed/ + subdirectory. These are sorted chronologically by their YYYYMMDDHHMM- + timestamp prefix so that the importer always sees the earliest snapshot + first, which is critical for correct incremental scrobbling since + retroarch overwrites the same .lrtl filename. """ import hashlib import shutil + import re import zipfile from datetime import datetime @@ -343,8 +355,9 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): except Exception: pass + # Collect files from root directory try: - files = webdav_client.list(retroarch_path, get_info=True) + root_files = webdav_client.list(retroarch_path, get_info=True) except Exception as e: logger.warning( "Could not list var/retroarch/", @@ -353,18 +366,50 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): return 0 # Extract basename from path since the webdav adapter returns name=None - for f in files: + for f in root_files: if f.get("path") and not f.get("name"): f["name"] = os.path.basename(f["path"]) - lrtl_files = sorted( - [f for f in files if (f.get("name") or "").lower().endswith(".lrtl")], - key=lambda x: (x.get("name") or ""), - ) + lrtl_files = [ + f for f in root_files + if (f.get("name") or "").lower().endswith(".lrtl") + ] + + # Optionally include historical files from processed/ + if include_processed: + try: + processed_files = webdav_client.list(processed_dir, get_info=True) + except Exception as e: + logger.warning( + "Could not list var/retroarch/processed/", + extra={"user_id": user_id, "error": str(e)}, + ) + else: + for f in processed_files: + if f.get("path") and not f.get("name"): + f["name"] = os.path.basename(f["path"]) + lrtl_files.extend([ + f for f in processed_files + if (f.get("name") or "").lower().endswith(".lrtl") + ]) + if not lrtl_files: logger.info("No .lrtl files found on webdav", extra={"user_id": user_id}) return 0 + # Sort chronologically: processed files (timestamp prefixed) first, + # current root files last. + ts_pattern = re.compile(r"^(\d{12})-") + def sort_key(f): + name = f.get("name") or "" + m = ts_pattern.match(name) + if m: + return m.group(1) + # Root files (no timestamp) go after all historical snapshots + return "999999999999" + + lrtl_files.sort(key=sort_key) + # Don't queue if one is still being processed if RetroarchImport.objects.filter( user_id=user_id, processed_finished__isnull=True @@ -382,7 +427,6 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): hasher.update((f.get("etag") or f.get("modified") or "").encode()) content_hash = hasher.hexdigest() - # Skip if the last completed import already has this hash last_import = ( RetroarchImport.objects.filter( user_id=user_id, processed_finished__isnull=False @@ -390,7 +434,9 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): .order_by("-processed_finished") .first() ) - if last_import and last_import.files_hash == content_hash: + + # Skip unchanged root files only, never skip when re-processing history + if not include_processed and last_import and last_import.files_hash == content_hash: logger.info( "Retroarch lrtl files unchanged for user, skipping", extra={"user_id": user_id}, @@ -414,7 +460,11 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): imported_filenames = [] for f in lrtl_files: basename = f.get("name") - remote_path = f"{retroarch_path}{basename}" + # Determine source directory: processed files already live under processed/ + is_processed = ts_pattern.match(basename) + source_dir = processed_dir if is_processed else retroarch_path + + remote_path = f"{source_dir}{basename}" dst = os.path.join(download_dir, basename) try: webdav_client.download_sync( @@ -423,13 +473,16 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): ) downloaded.append(basename) - ts = datetime.now().strftime("%Y%m%d%H%M%S") - processed_name = f"{ts}-{basename}" - imported_filenames.append(processed_name) - webdav_client.move( - remote_path, - f"{processed_dir}{processed_name}", - ) + if is_processed: + imported_filenames.append(basename) + else: + ts = datetime.now().strftime("%Y%m%d%H%M%S") + processed_name = f"{ts}-{basename}" + imported_filenames.append(processed_name) + webdav_client.move( + remote_path, + f"{processed_dir}{processed_name}", + ) except Exception as e: logger.error("Failed to download retroarch %s: %s", basename, e)