[importers] Try speeding up koreader imports

vrobbler/apps/scrobbles/importers/webdav.py

@@ -57,7 +57,13 @@ def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False)
         scale_count += scan_webdav_for_scale(client, user_id)
 
     logger.info(
-        "WebDAV import complete",
+        "Started %d KOReader, %d Trail GPX, %d Retroarch, %d BGStats, %d eBird, %d Scale WebDAV imports",
+        ko_count,
+        gpx_count,
+        retro_count,
+        bgstats_count,
+        ebird_count,
+        scale_count,
         extra={
             "koreader": ko_count,
             "trail_gpx": gpx_count,
@@ -76,17 +82,39 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
     Uses the remote file's ETag (from a single PROPFIND) to detect changes
     without downloading the file, making the common no-change case fast.
     """
-    koreader_path = DEFAULT_KOREADER_PATH + "statistics.sqlite3"
+    # Don't queue if one is still being processed
-    try:
+    if KoReaderImport.objects.filter(
-        remote_info = webdav_client.info(koreader_path)
+        user_id=user_id, processed_finished__isnull=True
-    except:
+    ).exists():
         logger.info(
-            "No koreader stats file found on webdav",
+            "KoReader import already pending for user",
             extra={"user_id": user_id},
         )
         return 0
 
-    remote_etag = remote_info.get("etag") or ""
+    try:
+        files = webdav_client.list(DEFAULT_KOREADER_PATH, get_info=True)
+    except:
+        logger.info(
+            "No var/koreader/ directory on webdav",
+            extra={"user_id": user_id},
+        )
+        return 0
+
+    sqlite_info = None
+    for f in files:
+        if (f.get("name") or "") == "statistics.sqlite3":
+            sqlite_info = f
+            break
+
+    if not sqlite_info:
+        logger.info(
+            "No statistics.sqlite3 found in var/koreader/",
+            extra={"user_id": user_id},
+        )
+        return 0
+
+    remote_etag = sqlite_info.get("etag") or ""
 
     last_import = (
         KoReaderImport.objects.filter(user_id=user_id, processed_finished__isnull=False)
@@ -117,7 +145,7 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
     new_hash = get_file_md5_hash(file_path)
     old_hash = last_import.file_md5_hash() if last_import else None
 
-    if last_import and remote_etag and new_hash == old_hash:
+    if last_import and not remote_etag and new_hash == old_hash:
         logger.info(
             "koreader stats file has not changed (content hash match)",
             extra={

vrobbler/apps/scrobbles/management/commands/import_from_webdav.py

@@ -21,14 +21,7 @@ class Command(BaseCommand):
         if options["restart"]:
             restart = True
         update_hash = options.get("update_retroarch_hash", False)
-        ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count = (
+        webdav.import_from_webdav_for_all_users(
-            webdav.import_from_webdav_for_all_users(
+            restart=restart,
-                restart=restart,
+            update_retroarch_hash=update_hash,
-                update_retroarch_hash=update_hash,
-            )
-        )
-        print(
-            f"Started {ko_count} KOReader, {gpx_count} Trail GPX, "
-            f"{retro_count} Retroarch, {bgstats_count} BGStats, "
-            f"{ebird_count} eBird, {scale_count} Scale WebDAV imports"
         )