[tasks] Add auto importing of scale csv from webdav

vrobbler/apps/scrobbles/importers/webdav.py

@@ -18,6 +18,7 @@ DEFAULT_KOREADER_PATH = "var/koreader/"
 DEFAULT_RETROARCH_PATH = "var/retroarch/"
 DEFAULT_BGSTATS_PATH = "var/bgstats/"
 DEFAULT_EBIRD_PATH = "var/ebird/"
+DEFAULT_SCALE_PATH = "var/scale/"
 
 
 def import_from_webdav_for_all_users(restart=False):
@@ -35,6 +36,7 @@ def import_from_webdav_for_all_users(restart=False):
     retro_count = 0
     bgstats_count = 0
     ebird_count = 0
+    scale_count = 0
 
     for user_id in webdav_enabled_user_ids:
         client = get_webdav_client(user_id)
@@ -43,6 +45,7 @@ def import_from_webdav_for_all_users(restart=False):
         retro_count += scan_webdav_for_retroarch(client, user_id)
         bgstats_count += scan_webdav_for_bgstats(client, user_id)
         ebird_count += scan_webdav_for_ebird(client, user_id)
+        scale_count += scan_webdav_for_scale(client, user_id)
 
     logger.info(
         "WebDAV import complete",
@@ -52,9 +55,10 @@ def import_from_webdav_for_all_users(restart=False):
             "retroarch": retro_count,
             "bgstats": bgstats_count,
             "ebird": ebird_count,
+            "scale": scale_count,
         },
     )
-    return ko_count, gpx_count, retro_count, bgstats_count, ebird_count
+    return ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count
 
 
 def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
@@ -70,9 +74,7 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
         return 0
 
     last_import = (
-        KoReaderImport.objects.filter(
-            user_id=user_id, processed_finished__isnull=False
-        )
+        KoReaderImport.objects.filter(user_id=user_id, processed_finished__isnull=False)
         .order_by("processed_finished")
         .last()
     )
@@ -184,9 +186,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
     try:
         webdav_client.info(retroarch_path)
     except:
-        logger.info(
-            "No var/retroarch/ directory on webdav", extra={"user_id": user_id}
-        )
+        logger.info("No var/retroarch/ directory on webdav", extra={"user_id": user_id})
         return 0
 
     try:
@@ -199,9 +199,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
         return 0
 
     lrtl_basenames = sorted(
-        os.path.basename(fname)
-        for fname in files
-        if fname.lower().endswith(".lrtl")
+        os.path.basename(fname) for fname in files if fname.lower().endswith(".lrtl")
     )
     if not lrtl_basenames:
         logger.info("No .lrtl files found on webdav", extra={"user_id": user_id})
@@ -229,9 +227,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
                 )
                 downloaded.append(basename)
             except Exception as e:
-                logger.error(
-                    "Failed to download retroarch %s: %s", basename, e
-                )
+                logger.error("Failed to download retroarch %s: %s", basename, e)
 
         if not downloaded:
             return 0
@@ -263,7 +259,8 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
             return 0
 
         zip_path = os.path.join(
-            download_dir, f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip"
+            download_dir,
+            f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip",
         )
         with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
             for basename in downloaded:
@@ -275,9 +272,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
             files_hash=content_hash,
         )
         with open(zip_path, "rb") as f:
-            imp.lrtl_file.save(
-                f"retroarch-batch-{imp.uuid}.zip", f, save=True
-            )
+            imp.lrtl_file.save(f"retroarch-batch-{imp.uuid}.zip", f, save=True)
         process_retroarch_import.delay(imp.id)
         logger.info(
             "Queued retroarch import %s with %d file(s) (hash=%s)",
@@ -404,3 +399,62 @@ def scan_webdav_for_ebird(webdav_client, user_id):
             os.unlink(tmp.name)
 
     return new_imports
+
+
+def scan_webdav_for_scale(webdav_client, user_id):
+    """Download .csv files from WebDAV var/scale/ and queue imports for new files."""
+    from scrobbles.models import ScaleCSVImport
+    from scrobbles.tasks import process_scale_csv_import
+
+    scale_path = (
+        DEFAULT_SCALE_PATH  # TODO Allow this to be configured in a user profile setting
+    )
+    try:
+        webdav_client.info(scale_path)
+    except:
+        logger.info("No var/scale/ directory on webdav", extra={"user_id": user_id})
+        return 0
+
+    try:
+        files = webdav_client.list(scale_path)
+    except Exception as e:
+        logger.warning(
+            "Could not list var/scale/",
+            extra={"user_id": user_id, "error": str(e)},
+        )
+        return 0
+
+    new_imports = 0
+    already_imported = set(
+        ScaleCSVImport.objects.filter(user_id=user_id).values_list(
+            "original_filename", flat=True
+        )
+    )
+
+    for fname in files:
+        fname = os.path.basename(fname)
+        if not fname.lower().endswith(".csv"):
+            continue
+        if fname in already_imported:
+            logger.debug(f"Skipping already-imported {fname}")
+            continue
+
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
+        try:
+            webdav_client.download_sync(
+                remote_path=f"{scale_path}/{fname}", local_path=tmp.name
+            )
+            imp = ScaleCSVImport.objects.create(
+                user_id=user_id,
+                original_filename=fname,
+            )
+            with open(tmp.name, "rb") as f:
+                imp.csv_file.save(fname, f, save=True)
+            process_scale_csv_import.delay(imp.id)
+            new_imports += 1
+        except Exception as e:
+            logger.error(f"Failed to import Scale CSV file {fname}: {e}")
+        finally:
+            os.unlink(tmp.name)
+
+    return new_imports

vrobbler/apps/scrobbles/management/commands/import_from_webdav.py

@@ -14,11 +14,11 @@ class Command(BaseCommand):
         restart = False
         if options["restart"]:
             restart = True
-        ko_count, gpx_count, retro_count, bgstats_count, ebird_count = (
+        ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count = (
             webdav.import_from_webdav_for_all_users(restart=restart)
         )
         print(
             f"Started {ko_count} KOReader, {gpx_count} Trail GPX, "
             f"{retro_count} Retroarch, {bgstats_count} BGStats, "
-            f"{ebird_count} eBird WebDAV imports"
+            f"{ebird_count} eBird, {scale_count} Scale WebDAV imports"
         )

vrobbler/apps/scrobbles/migrations/0085_scale_csv_import_original_filename.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.29 on 2026-05-24 15:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0084_fix_ebird_sequence"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scalecsvimport",
+            name="original_filename",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -285,6 +285,7 @@ class ScaleCSVImport(BaseFileImportMixin):
         return path
 
     csv_file = models.FileField(upload_to=get_path, **BNULL)
+    original_filename = models.CharField(max_length=255, **BNULL)
 
     def process(self, force=False):
         from scrobbles.importers.scale import import_scale_csv

vrobbler/apps/scrobbles/tasks.py

@@ -168,6 +168,16 @@ def process_ebird_csv_import(import_id):
     birding_import.process()
 
 
+@shared_task
+def process_scale_csv_import(import_id):
+    ScaleCSVImport = apps.get_model("scrobbles", "ScaleCSVImport")
+    scale_import = ScaleCSVImport.objects.filter(id=import_id).first()
+    if not scale_import:
+        logger.warn(f"ScaleCSVImport not found with id {import_id}")
+        return
+    scale_import.process()
+
+
 @shared_task
 def create_yesterdays_charts():
     """Build/update charts for all users starting from last known record."""

vrobbler/apps/scrobbles/views.py

@@ -622,6 +622,9 @@ class ScaleCSVImportCreateView(
     def form_valid(self, form):
         self.object = form.save(commit=False)
         self.object.user = self.request.user
+        self.object.original_filename = (
+            form.cleaned_data["csv_file"].name
+        )
         self.object.save()
         self.object.process()
         return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))