From 62d8ffd794e34f0f5352aabaee04c4e28a4f854c Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Thu, 28 May 2026 17:19:08 -0400 Subject: [PATCH] [importers] Add flag to reimport already processed files --- PROJECT.org | 42 ++++++++++++++- vrobbler/apps/scrobbles/importers/webdav.py | 52 +++++++++++++++++-- .../management/commands/import_from_webdav.py | 9 ++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/PROJECT.org b/PROJECT.org index 76cd8e1..d25dc49 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -93,7 +93,7 @@ fetching and simple saving. :LOGBOOK: CLOCK: [2025-07-09 Wed 09:55]--[2025-07-09 Wed 10:15] => 0:20 :END: -* Backlog [26/42] :vrobbler:project:personal: +* Backlog [28/44] :vrobbler:project:personal: ** TODO [#C] Add sentiment parsing for Scrobbles with notes :vrobbler:project:scrobbles:sentiment: :PROPERTIES: :ID: 37781d6a-f3b0-48b2-bf98-33c2c791cf85 @@ -496,7 +496,45 @@ or at least make it optional, and then require a M2M between Track and Artist. Then this one would be a Track by both `Matt Sweeney` and `Bonnie "Prince" Billy` -** TODO [#A] Allow special parameter to re-import already processed GPX files :imports:gpx: +** TODO [#A] Move imported eBird CSV files to processed/ directory on WebDAV :webdav:ebird:importers: +:PROPERTIES: +:ID: 445e1253-d353-4b55-b1d8-39d0a0dcdd34 +:END: + - File: ~vrobbler/apps/scrobbles/importers/webdav.py~ (line 439) + - Same pattern as the GPX importer: after importing a =.csv= file from + WebDAV, move it to =var/ebird/processed/= with a timestamp appended. + +** TODO [#A] Move imported Scale CSV files to processed/ directory on WebDAV :webdav:scale:importers: +:PROPERTIES: +:ID: 1a0de363-d1ea-466e-9966-e24941a6180b +:END: + - File: ~vrobbler/apps/scrobbles/importers/webdav.py~ (line 496) + - Same pattern as the GPX importer: after importing a =.csv= file from + WebDAV, move it to =var/scale/processed/= with a timestamp appended. + +** DONE [#A] Allow special parameter to re-import already processed GPX files :imports:gpx: +:PROPERTIES: +:ID: 166c0809-c11a-4d02-8071-7a69dcb36e64 +:END: + +*** Description + +Now that we stash imported GPX files in the processed/ subdirectory on import, +it would be nice to have a flag like --include-processed on the webdav importer +that would import files both in the root of gpx/ and also in the processed +directory. This would aide testing imports in staging quickly without constantly +moving files back and forth. + +** DONE [#A] Move imported GPX files to processed/ directory on WebDAV :webdav:gpx:importers: +:PROPERTIES: +:ID: db2b02fc-817c-4c39-bd51-13f9b77c7888 +:END: + - File: ~vrobbler/apps/scrobbles/importers/webdav.py~ (line 198) + - After importing a GPX/FIT file from WebDAV, move it to a =processed/= + subdirectory with a timestamp appended. This eliminates the DB lookup for + already-imported filenames — any file present in the top-level directory + is new. Also makes manual re-imports easy (just move a file back). + ** DONE [#A] Add CSS Grid calendar view for scrobbles :vrobbler:personal:project:templates:feature: :PROPERTIES: :ID: be915acf-d803-466a-8770-823819ebf2a9 diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index 9870ee2..8b68773 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -22,7 +22,10 @@ DEFAULT_SCALE_PATH = "var/scale/" def import_from_webdav_for_all_users( - restart=False, update_retroarch_hash=False, update_koreader_etag=False + restart=False, + update_retroarch_hash=False, + update_koreader_etag=False, + include_processed=False, ): """Iterate all WebDAV-enabled users, scanning each media-type directory.""" webdav_enabled_user_ids = UserProfile.objects.filter( @@ -31,6 +34,12 @@ def import_from_webdav_for_all_users( webdav_pass__isnull=False, webdav_auto_import=True, ).values_list("user_id", flat=True) + if include_processed: + logger.warning( + "Re-importing previously-processed files from processed/ subdirectories " + "— this may create duplicate scrobbles." + ) + logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts") ko_count = 0 @@ -49,7 +58,9 @@ def import_from_webdav_for_all_users( client, user_id, restart, update_etag_only=update_koreader_etag ) logger.info("Scanning WebDAV gpx for user %s", user_id) - gpx_count += scan_webdav_for_gpx(client, user_id) + gpx_count += scan_webdav_for_gpx( + client, user_id, include_processed=include_processed + ) logger.info("Scanning WebDAV retroarch for user %s", user_id) retro_count += scan_webdav_for_retroarch( client, user_id, update_hash_only=update_retroarch_hash @@ -200,7 +211,7 @@ def scan_webdav_for_koreader( return 1 -def scan_webdav_for_gpx(webdav_client, user_id): +def scan_webdav_for_gpx(webdav_client, user_id, include_processed=False): gpx_path = DEFAULT_GPX_PATH # TODO allow this to be configured in user settings try: webdav_client.info(DEFAULT_GPX_PATH) @@ -259,6 +270,41 @@ def scan_webdav_for_gpx(webdav_client, user_id): finally: os.unlink(tmp.name) + if include_processed: + try: + processed_files = webdav_client.list(processed_dir) + except Exception as e: + logger.warning( + "Could not list var/gpx/processed/", + extra={"user_id": user_id, "error": str(e)}, + ) + return new_imports + + for fname in processed_files: + fname = os.path.basename(fname) + if not fname.lower().endswith(gpx_extensions): + continue + + remote_path = f"{processed_dir}{fname}" + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname) + try: + webdav_client.download_sync( + remote_path=remote_path, local_path=tmp.name + ) + imp = TrailGPXImport.objects.create( + user_id=user_id, + original_filename=fname, + ) + with open(tmp.name, "rb") as f: + imp.gpx_file.save(fname, f, save=True) + + process_trail_gpx_import.delay(imp.id) + new_imports += 1 + except Exception as e: + logger.error(f"Failed to import processed GPX file {fname}: {e}") + finally: + os.unlink(tmp.name) + return new_imports diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py index f06d192..4c01f32 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from vrobbler.apps.scrobbles.importers import webdav @@ -21,6 +22,12 @@ class Command(BaseCommand): help="Store current WebDAV ETag on last KoReader import " "without re-importing (migration helper)", ) + parser.add_argument( + "--include-processed", + action="store_true", + help="Also import files already moved to processed/ subdirectories " + "(may produce duplicate scrobbles; use with care on production)", + ) def handle(self, *args, **options): restart = False @@ -28,8 +35,10 @@ class Command(BaseCommand): restart = True update_hash = options.get("update_retroarch_hash", False) update_etag = options.get("update_koreader_etag", False) + include_processed = options.get("include_processed", False) webdav.import_from_webdav_for_all_users( restart=restart, update_retroarch_hash=update_hash, update_koreader_etag=update_etag, + include_processed=include_processed, )