[tasks] Add auto importing of scale csv from webdav

vrobbler/apps/scrobbles/importers/webdav.py

@@ -18,6 +18,7 @@ DEFAULT_KOREADER_PATH = "var/koreader/"
 DEFAULT_RETROARCH_PATH = "var/retroarch/"
 DEFAULT_BGSTATS_PATH = "var/bgstats/"
 DEFAULT_EBIRD_PATH = "var/ebird/"
+DEFAULT_SCALE_PATH = "var/scale/"
 
 
 def import_from_webdav_for_all_users(restart=False):
@@ -35,6 +36,7 @@ def import_from_webdav_for_all_users(restart=False):
     retro_count = 0
     bgstats_count = 0
     ebird_count = 0
+    scale_count = 0
 
     for user_id in webdav_enabled_user_ids:
         client = get_webdav_client(user_id)
@@ -43,6 +45,7 @@ def import_from_webdav_for_all_users(restart=False):
         retro_count += scan_webdav_for_retroarch(client, user_id)
         bgstats_count += scan_webdav_for_bgstats(client, user_id)
         ebird_count += scan_webdav_for_ebird(client, user_id)
+        scale_count += scan_webdav_for_scale(client, user_id)
 
     logger.info(
         "WebDAV import complete",
@@ -52,9 +55,10 @@ def import_from_webdav_for_all_users(restart=False):
             "retroarch": retro_count,
             "bgstats": bgstats_count,
             "ebird": ebird_count,
+            "scale": scale_count,
         },
     )
-    return ko_count, gpx_count, retro_count, bgstats_count, ebird_count
+    return ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count
 
 
 def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
@@ -70,9 +74,7 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False):
         return 0
 
     last_import = (
-        KoReaderImport.objects.filter(
-            user_id=user_id, processed_finished__isnull=False
-        )
+        KoReaderImport.objects.filter(user_id=user_id, processed_finished__isnull=False)
         .order_by("processed_finished")
         .last()
     )
@@ -184,9 +186,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
     try:
         webdav_client.info(retroarch_path)
     except:
-        logger.info(
-            "No var/retroarch/ directory on webdav", extra={"user_id": user_id}
-        )
+        logger.info("No var/retroarch/ directory on webdav", extra={"user_id": user_id})
         return 0
 
     try:
@@ -199,9 +199,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
         return 0
 
     lrtl_basenames = sorted(
-        os.path.basename(fname)
-        for fname in files
-        if fname.lower().endswith(".lrtl")
+        os.path.basename(fname) for fname in files if fname.lower().endswith(".lrtl")
     )
     if not lrtl_basenames:
         logger.info("No .lrtl files found on webdav", extra={"user_id": user_id})
@@ -229,9 +227,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
                 )
                 downloaded.append(basename)
             except Exception as e:
-                logger.error(
-                    "Failed to download retroarch %s: %s", basename, e
-                )
+                logger.error("Failed to download retroarch %s: %s", basename, e)
 
         if not downloaded:
             return 0
@@ -263,7 +259,8 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
             return 0
 
         zip_path = os.path.join(
-            download_dir, f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip"
+            download_dir,
+            f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip",
         )
         with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
             for basename in downloaded:
@@ -275,9 +272,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id):
             files_hash=content_hash,
         )
         with open(zip_path, "rb") as f:
-            imp.lrtl_file.save(
-                f"retroarch-batch-{imp.uuid}.zip", f, save=True
-            )
+            imp.lrtl_file.save(f"retroarch-batch-{imp.uuid}.zip", f, save=True)
         process_retroarch_import.delay(imp.id)
         logger.info(
             "Queued retroarch import %s with %d file(s) (hash=%s)",
@@ -404,3 +399,62 @@ def scan_webdav_for_ebird(webdav_client, user_id):
             os.unlink(tmp.name)
 
     return new_imports
+
+
+def scan_webdav_for_scale(webdav_client, user_id):
+    """Download .csv files from WebDAV var/scale/ and queue imports for new files."""
+    from scrobbles.models import ScaleCSVImport
+    from scrobbles.tasks import process_scale_csv_import
+
+    scale_path = (
+        DEFAULT_SCALE_PATH  # TODO Allow this to be configured in a user profile setting
+    )
+    try:
+        webdav_client.info(scale_path)
+    except:
+        logger.info("No var/scale/ directory on webdav", extra={"user_id": user_id})
+        return 0
+
+    try:
+        files = webdav_client.list(scale_path)
+    except Exception as e:
+        logger.warning(
+            "Could not list var/scale/",
+            extra={"user_id": user_id, "error": str(e)},
+        )
+        return 0
+
+    new_imports = 0
+    already_imported = set(
+        ScaleCSVImport.objects.filter(user_id=user_id).values_list(
+            "original_filename", flat=True
+        )
+    )
+
+    for fname in files:
+        fname = os.path.basename(fname)
+        if not fname.lower().endswith(".csv"):
+            continue
+        if fname in already_imported:
+            logger.debug(f"Skipping already-imported {fname}")
+            continue
+
+        tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
+        try:
+            webdav_client.download_sync(
+                remote_path=f"{scale_path}/{fname}", local_path=tmp.name
+            )
+            imp = ScaleCSVImport.objects.create(
+                user_id=user_id,
+                original_filename=fname,
+            )
+            with open(tmp.name, "rb") as f:
+                imp.csv_file.save(fname, f, save=True)
+            process_scale_csv_import.delay(imp.id)
+            new_imports += 1
+        except Exception as e:
+            logger.error(f"Failed to import Scale CSV file {fname}: {e}")
+        finally:
+            os.unlink(tmp.name)
+
+    return new_imports