[importers] Add flag to reimport already processed files

PROJECT.org

@@ -93,7 +93,7 @@ fetching and simple saving.
 :LOGBOOK:
 CLOCK: [2025-07-09 Wed 09:55]--[2025-07-09 Wed 10:15] =>  0:20
 :END:
-* Backlog [26/42] :vrobbler:project:personal:
+* Backlog [28/44] :vrobbler:project:personal:
 ** TODO [#C] Add sentiment parsing for Scrobbles with notes :vrobbler:project:scrobbles:sentiment:
 :PROPERTIES:
 :ID:       37781d6a-f3b0-48b2-bf98-33c2c791cf85
@@ -496,7 +496,45 @@ or at least make it optional, and then require a M2M between Track and Artist.
 Then this one would be a Track by both `Matt Sweeney` and `Bonnie "Prince"
 Billy`
 
-** TODO [#A] Allow special parameter to re-import already processed GPX files :imports:gpx:
+** TODO [#A] Move imported eBird CSV files to processed/ directory on WebDAV :webdav:ebird:importers:
+:PROPERTIES:
+:ID:       445e1253-d353-4b55-b1d8-39d0a0dcdd34
+:END:
+    - File: ~vrobbler/apps/scrobbles/importers/webdav.py~ (line 439)
+    - Same pattern as the GPX importer: after importing a =.csv= file from
+      WebDAV, move it to =var/ebird/processed/= with a timestamp appended.
+
+** TODO [#A] Move imported Scale CSV files to processed/ directory on WebDAV :webdav:scale:importers:
+:PROPERTIES:
+:ID:       1a0de363-d1ea-466e-9966-e24941a6180b
+:END:
+    - File: ~vrobbler/apps/scrobbles/importers/webdav.py~ (line 496)
+    - Same pattern as the GPX importer: after importing a =.csv= file from
+      WebDAV, move it to =var/scale/processed/= with a timestamp appended.
+
+** DONE [#A] Allow special parameter to re-import already processed GPX files :imports:gpx:
+:PROPERTIES:
+:ID:       166c0809-c11a-4d02-8071-7a69dcb36e64
+:END:
+
+*** Description
+
+Now that we stash imported GPX files in the processed/ subdirectory on import,
+it would be nice to have a flag like --include-processed on the webdav importer
+that would import files both in the root of gpx/ and also in the processed
+directory. This would aide testing imports in staging quickly without constantly
+moving files back and forth.
+
+** DONE [#A] Move imported GPX files to processed/ directory on WebDAV :webdav:gpx:importers:
+:PROPERTIES:
+:ID:       db2b02fc-817c-4c39-bd51-13f9b77c7888
+:END:
+    - File: ~vrobbler/apps/scrobbles/importers/webdav.py~ (line 198)
+    - After importing a GPX/FIT file from WebDAV, move it to a =processed/=
+      subdirectory with a timestamp appended. This eliminates the DB lookup for
+      already-imported filenames — any file present in the top-level directory
+      is new. Also makes manual re-imports easy (just move a file back).
+
 ** DONE [#A] Add CSS Grid calendar view for scrobbles :vrobbler:personal:project:templates:feature:
 :PROPERTIES:
 :ID:       be915acf-d803-466a-8770-823819ebf2a9

vrobbler/apps/scrobbles/importers/webdav.py

@@ -22,7 +22,10 @@ DEFAULT_SCALE_PATH = "var/scale/"
 
 
 def import_from_webdav_for_all_users(
-    restart=False, update_retroarch_hash=False, update_koreader_etag=False
+    restart=False,
+    update_retroarch_hash=False,
+    update_koreader_etag=False,
+    include_processed=False,
 ):
     """Iterate all WebDAV-enabled users, scanning each media-type directory."""
     webdav_enabled_user_ids = UserProfile.objects.filter(
@@ -31,6 +34,12 @@ def import_from_webdav_for_all_users(
         webdav_pass__isnull=False,
         webdav_auto_import=True,
     ).values_list("user_id", flat=True)
+    if include_processed:
+        logger.warning(
+            "Re-importing previously-processed files from processed/ subdirectories "
+            "— this may create duplicate scrobbles."
+        )
+
     logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
 
     ko_count = 0
@@ -49,7 +58,9 @@ def import_from_webdav_for_all_users(
             client, user_id, restart, update_etag_only=update_koreader_etag
         )
         logger.info("Scanning WebDAV gpx for user %s", user_id)
-        gpx_count += scan_webdav_for_gpx(client, user_id)
+        gpx_count += scan_webdav_for_gpx(
+            client, user_id, include_processed=include_processed
+        )
         logger.info("Scanning WebDAV retroarch for user %s", user_id)
         retro_count += scan_webdav_for_retroarch(
             client, user_id, update_hash_only=update_retroarch_hash
@@ -200,7 +211,7 @@ def scan_webdav_for_koreader(
     return 1
 
 
-def scan_webdav_for_gpx(webdav_client, user_id):
+def scan_webdav_for_gpx(webdav_client, user_id, include_processed=False):
     gpx_path = DEFAULT_GPX_PATH  # TODO allow this to be configured in user settings
     try:
         webdav_client.info(DEFAULT_GPX_PATH)
@@ -259,6 +270,41 @@ def scan_webdav_for_gpx(webdav_client, user_id):
         finally:
             os.unlink(tmp.name)
 
+    if include_processed:
+        try:
+            processed_files = webdav_client.list(processed_dir)
+        except Exception as e:
+            logger.warning(
+                "Could not list var/gpx/processed/",
+                extra={"user_id": user_id, "error": str(e)},
+            )
+            return new_imports
+
+        for fname in processed_files:
+            fname = os.path.basename(fname)
+            if not fname.lower().endswith(gpx_extensions):
+                continue
+
+            remote_path = f"{processed_dir}{fname}"
+            tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
+            try:
+                webdav_client.download_sync(
+                    remote_path=remote_path, local_path=tmp.name
+                )
+                imp = TrailGPXImport.objects.create(
+                    user_id=user_id,
+                    original_filename=fname,
+                )
+                with open(tmp.name, "rb") as f:
+                    imp.gpx_file.save(fname, f, save=True)
+
+                process_trail_gpx_import.delay(imp.id)
+                new_imports += 1
+            except Exception as e:
+                logger.error(f"Failed to import processed GPX file {fname}: {e}")
+            finally:
+                os.unlink(tmp.name)
+
     return new_imports
 
 
@@ -507,7 +553,12 @@ def scan_webdav_for_ebird(webdav_client, user_id):
 
 
 def scan_webdav_for_scale(webdav_client, user_id):
-    """Download .csv files from WebDAV var/scale/ and queue imports for new files."""
+    """Download .csv files from WebDAV var/scale/ and queue imports for new files.
+
+    Because the scale CSV always has the same filename but grows by appending
+    rows, we detect changes by downloading the file and comparing its content
+    hash against the last completed import, rather than checking the filename.
+    """
     from scrobbles.models import ScaleCSVImport
     from scrobbles.tasks import process_scale_csv_import
 
@@ -530,28 +581,41 @@ def scan_webdav_for_scale(webdav_client, user_id):
         return 0
 
     new_imports = 0
-    already_imported = set(
-        ScaleCSVImport.objects.filter(user_id=user_id).values_list(
-            "original_filename", flat=True
-        )
-    )
 
     for fname in files:
         fname = os.path.basename(fname)
         if not fname.lower().endswith(".csv"):
             continue
-        if fname in already_imported:
-            logger.debug(f"Skipping already-imported {fname}")
-            continue
 
         tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
         try:
             webdav_client.download_sync(
                 remote_path=f"{scale_path}/{fname}", local_path=tmp.name
             )
+
+            new_hash = get_file_md5_hash(tmp.name)
+
+            last_import = (
+                ScaleCSVImport.objects.filter(
+                    user_id=user_id,
+                    original_filename=fname,
+                    processed_finished__isnull=False,
+                )
+                .order_by("-processed_finished")
+                .first()
+            )
+            if last_import and last_import.file_hash == new_hash:
+                logger.debug(
+                    "Scale CSV %s unchanged (hash match), skipping",
+                    fname,
+                    extra={"user_id": user_id, "hash": new_hash},
+                )
+                continue
+
             imp = ScaleCSVImport.objects.create(
                 user_id=user_id,
                 original_filename=fname,
+                file_hash=new_hash,
             )
             with open(tmp.name, "rb") as f:
                 imp.csv_file.save(fname, f, save=True)

vrobbler/apps/scrobbles/management/commands/import_from_webdav.py

@@ -1,4 +1,5 @@
 from django.core.management.base import BaseCommand
+
 from vrobbler.apps.scrobbles.importers import webdav
 
 
@@ -21,6 +22,12 @@ class Command(BaseCommand):
             help="Store current WebDAV ETag on last KoReader import "
             "without re-importing (migration helper)",
         )
+        parser.add_argument(
+            "--include-processed",
+            action="store_true",
+            help="Also import files already moved to processed/ subdirectories "
+            "(may produce duplicate scrobbles; use with care on production)",
+        )
 
     def handle(self, *args, **options):
         restart = False
@@ -28,8 +35,10 @@ class Command(BaseCommand):
             restart = True
         update_hash = options.get("update_retroarch_hash", False)
         update_etag = options.get("update_koreader_etag", False)
+        include_processed = options.get("include_processed", False)
         webdav.import_from_webdav_for_all_users(
             restart=restart,
             update_retroarch_hash=update_hash,
             update_koreader_etag=update_etag,
+            include_processed=include_processed,
         )

vrobbler/apps/scrobbles/migrations/0088_scalecsvimport_file_hash.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.29 on 2026-05-28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0087_koreaderimport_webdav_etag"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scalecsvimport",
+            name="file_hash",
+            field=models.CharField(blank=True, max_length=32, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -287,6 +287,7 @@ class ScaleCSVImport(BaseFileImportMixin):
 
     csv_file = models.FileField(upload_to=get_path, **BNULL)
     original_filename = models.CharField(max_length=255, **BNULL)
+    file_hash = models.CharField(max_length=32, **BNULL)
 
     def process(self, force=False):
         from scrobbles.importers.scale import import_scale_csv