[importers] Add flag to reimport already processed files

vrobbler/apps/scrobbles/importers/webdav.py

@@ -22,7 +22,10 @@ DEFAULT_SCALE_PATH = "var/scale/"
 
 
 def import_from_webdav_for_all_users(
-    restart=False, update_retroarch_hash=False, update_koreader_etag=False
+    restart=False,
+    update_retroarch_hash=False,
+    update_koreader_etag=False,
+    include_processed=False,
 ):
     """Iterate all WebDAV-enabled users, scanning each media-type directory."""
     webdav_enabled_user_ids = UserProfile.objects.filter(
@@ -31,6 +34,12 @@ def import_from_webdav_for_all_users(
         webdav_pass__isnull=False,
         webdav_auto_import=True,
     ).values_list("user_id", flat=True)
+    if include_processed:
+        logger.warning(
+            "Re-importing previously-processed files from processed/ subdirectories "
+            "— this may create duplicate scrobbles."
+        )
+
     logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
 
     ko_count = 0
@@ -49,7 +58,9 @@ def import_from_webdav_for_all_users(
             client, user_id, restart, update_etag_only=update_koreader_etag
         )
         logger.info("Scanning WebDAV gpx for user %s", user_id)
-        gpx_count += scan_webdav_for_gpx(client, user_id)
+        gpx_count += scan_webdav_for_gpx(
+            client, user_id, include_processed=include_processed
+        )
         logger.info("Scanning WebDAV retroarch for user %s", user_id)
         retro_count += scan_webdav_for_retroarch(
             client, user_id, update_hash_only=update_retroarch_hash
@@ -200,7 +211,7 @@ def scan_webdav_for_koreader(
     return 1
 
 
-def scan_webdav_for_gpx(webdav_client, user_id):
+def scan_webdav_for_gpx(webdav_client, user_id, include_processed=False):
     gpx_path = DEFAULT_GPX_PATH  # TODO allow this to be configured in user settings
     try:
         webdav_client.info(DEFAULT_GPX_PATH)
@@ -259,6 +270,41 @@ def scan_webdav_for_gpx(webdav_client, user_id):
         finally:
             os.unlink(tmp.name)
 
+    if include_processed:
+        try:
+            processed_files = webdav_client.list(processed_dir)
+        except Exception as e:
+            logger.warning(
+                "Could not list var/gpx/processed/",
+                extra={"user_id": user_id, "error": str(e)},
+            )
+            return new_imports
+
+        for fname in processed_files:
+            fname = os.path.basename(fname)
+            if not fname.lower().endswith(gpx_extensions):
+                continue
+
+            remote_path = f"{processed_dir}{fname}"
+            tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
+            try:
+                webdav_client.download_sync(
+                    remote_path=remote_path, local_path=tmp.name
+                )
+                imp = TrailGPXImport.objects.create(
+                    user_id=user_id,
+                    original_filename=fname,
+                )
+                with open(tmp.name, "rb") as f:
+                    imp.gpx_file.save(fname, f, save=True)
+
+                process_trail_gpx_import.delay(imp.id)
+                new_imports += 1
+            except Exception as e:
+                logger.error(f"Failed to import processed GPX file {fname}: {e}")
+            finally:
+                os.unlink(tmp.name)
+
     return new_imports
 
 

vrobbler/apps/scrobbles/management/commands/import_from_webdav.py

@@ -1,4 +1,5 @@
 from django.core.management.base import BaseCommand
+
 from vrobbler.apps.scrobbles.importers import webdav
 
 
@@ -21,6 +22,12 @@ class Command(BaseCommand):
             help="Store current WebDAV ETag on last KoReader import "
             "without re-importing (migration helper)",
         )
+        parser.add_argument(
+            "--include-processed",
+            action="store_true",
+            help="Also import files already moved to processed/ subdirectories "
+            "(may produce duplicate scrobbles; use with care on production)",
+        )
 
     def handle(self, *args, **options):
         restart = False
@@ -28,8 +35,10 @@ class Command(BaseCommand):
             restart = True
         update_hash = options.get("update_retroarch_hash", False)
         update_etag = options.get("update_koreader_etag", False)
+        include_processed = options.get("include_processed", False)
         webdav.import_from_webdav_for_all_users(
             restart=restart,
             update_retroarch_hash=update_hash,
             update_koreader_etag=update_etag,
+            include_processed=include_processed,
         )