diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index e4b6cb5..6fe6122 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -71,10 +71,14 @@ def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False) def scan_webdav_for_koreader(webdav_client, user_id, restart=False): - """Check for koreader statistics.sqlite3 and queue import if changed.""" + """Check for koreader statistics.sqlite3 and queue import if changed. + + Uses the remote file's ETag (from a single PROPFIND) to detect changes + without downloading the file, making the common no-change case fast. + """ koreader_path = DEFAULT_KOREADER_PATH + "statistics.sqlite3" try: - webdav_client.info(koreader_path) + remote_info = webdav_client.info(koreader_path) except: logger.info( "No koreader stats file found on webdav", @@ -82,12 +86,26 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False): ) return 0 + remote_etag = remote_info.get("etag") or "" + last_import = ( KoReaderImport.objects.filter(user_id=user_id, processed_finished__isnull=False) .order_by("processed_finished") .last() ) + if last_import and last_import.webdav_etag and remote_etag: + if last_import.webdav_etag == remote_etag: + logger.info( + "koreader stats file unchanged (ETag match)", + extra={ + "user_id": user_id, + "etag": remote_etag, + "last_import_id": last_import.id, + }, + ) + return 0 + file_path = fetch_file_from_webdav(user_id) if not file_path: logger.warning( @@ -99,9 +117,9 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False): new_hash = get_file_md5_hash(file_path) old_hash = last_import.file_md5_hash() if last_import else None - if last_import and new_hash == old_hash: + if last_import and remote_etag and new_hash == old_hash: logger.info( - "koreader stats file has not changed", + "koreader stats file has not changed (content hash match)", extra={ "user_id": user_id, "new_hash": new_hash, @@ -124,6 +142,7 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False): os.unlink(file_path) return 0 + koreader_import.webdav_etag = remote_etag koreader_import.save_sqlite_file_to_self(file_path) process_koreader_import.delay(koreader_import.id) os.unlink(file_path) @@ -219,8 +238,8 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): return 0 lrtl_files = sorted( - [f for f in files if f.get("name", "").lower().endswith(".lrtl")], - key=lambda x: x.get("name", ""), + [f for f in files if (f.get("name") or "").lower().endswith(".lrtl")], + key=lambda x: (x.get("name") or ""), ) if not lrtl_files: logger.info("No .lrtl files found on webdav", extra={"user_id": user_id}) @@ -239,7 +258,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): # Compute hash from filenames + ETags (no downloads needed) hasher = hashlib.md5() for f in lrtl_files: - hasher.update(f.get("name", "").encode()) + hasher.update((f.get("name") or "").encode()) hasher.update((f.get("etag") or f.get("modified") or "").encode()) content_hash = hasher.hexdigest() diff --git a/vrobbler/apps/scrobbles/migrations/0087_koreaderimport_webdav_etag.py b/vrobbler/apps/scrobbles/migrations/0087_koreaderimport_webdav_etag.py new file mode 100644 index 0000000..112c59b --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0087_koreaderimport_webdav_etag.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.29 on 2026-05-26 13:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scrobbles", "0086_add_import_in_progress_to_lastfmimport"), + ] + + operations = [ + migrations.AddField( + model_name="koreaderimport", + name="webdav_etag", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 37282c5..2509bfa 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -188,6 +188,7 @@ class KoReaderImport(BaseFileImportMixin): return path sqlite_file = models.FileField(upload_to=get_path, **BNULL) + webdav_etag = models.CharField(max_length=255, **BNULL) def save_sqlite_file_to_self(self, file_path): with open(file_path, "rb") as f: