diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index 0460c0b..f7ee58c 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -20,7 +20,7 @@ DEFAULT_EBIRD_PATH = "var/ebird/" DEFAULT_SCALE_PATH = "var/scale/" -def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False): +def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False, update_koreader_etag=False): """Iterate all WebDAV-enabled users, scanning each media-type directory.""" webdav_enabled_user_ids = UserProfile.objects.filter( webdav_url__isnull=False, @@ -42,7 +42,9 @@ def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False) logger.info("Scanning WebDAV for user %s", user_id) logger.info("Scanning WebDAV koreader for user %s", user_id) - ko_count += scan_webdav_for_koreader(client, user_id, restart) + ko_count += scan_webdav_for_koreader( + client, user_id, restart, update_etag_only=update_koreader_etag + ) logger.info("Scanning WebDAV gpx for user %s", user_id) gpx_count += scan_webdav_for_gpx(client, user_id) logger.info("Scanning WebDAV retroarch for user %s", user_id) @@ -76,21 +78,26 @@ def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False) return ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count -def scan_webdav_for_koreader(webdav_client, user_id, restart=False): +def scan_webdav_for_koreader(webdav_client, user_id, restart=False, update_etag_only=False): """Check for koreader statistics.sqlite3 and queue import if changed. Uses the remote file's ETag (from a single PROPFIND) to detect changes without downloading the file, making the common no-change case fast. + + When *update_etag_only* is True, stores the remote ETag on the last + completed import without re-downloading — useful when migrating to the + ETag-based check to avoid a one-time re-import. """ - # Don't queue if one is still being processed - if KoReaderImport.objects.filter( - user_id=user_id, processed_finished__isnull=True - ).exists(): - logger.info( - "KoReader import already pending for user", - extra={"user_id": user_id}, - ) - return 0 + if not update_etag_only: + # Don't queue if one is still being processed + if KoReaderImport.objects.filter( + user_id=user_id, processed_finished__isnull=True + ).exists(): + logger.info( + "KoReader import already pending for user", + extra={"user_id": user_id}, + ) + return 0 try: files = webdav_client.list(DEFAULT_KOREADER_PATH, get_info=True) @@ -122,6 +129,17 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False): .last() ) + if update_etag_only: + if last_import and remote_etag: + last_import.webdav_etag = remote_etag + last_import.save(update_fields=["webdav_etag"]) + logger.info( + "Updated KoReader webdav_etag (%s) without re-import", + remote_etag[:16], + extra={"user_id": user_id, "last_import_id": last_import.id}, + ) + return 0 + if last_import and last_import.webdav_etag and remote_etag: if last_import.webdav_etag == remote_etag: logger.info( diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py index 3d00339..f06d192 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py @@ -15,13 +15,21 @@ class Command(BaseCommand): help="Update retroarch files_hash to new ETag-based scheme " "without re-importing (migration helper)", ) + parser.add_argument( + "--update-koreader-etag", + action="store_true", + help="Store current WebDAV ETag on last KoReader import " + "without re-importing (migration helper)", + ) def handle(self, *args, **options): restart = False if options["restart"]: restart = True update_hash = options.get("update_retroarch_hash", False) + update_etag = options.get("update_koreader_etag", False) webdav.import_from_webdav_for_all_users( restart=restart, update_retroarch_hash=update_hash, + update_koreader_etag=update_etag, )