diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index 736d997..71ae8d4 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -18,6 +18,7 @@ DEFAULT_KOREADER_PATH = "var/koreader/" DEFAULT_RETROARCH_PATH = "var/retroarch/" DEFAULT_BGSTATS_PATH = "var/bgstats/" DEFAULT_EBIRD_PATH = "var/ebird/" +DEFAULT_SCALE_PATH = "var/scale/" def import_from_webdav_for_all_users(restart=False): @@ -35,6 +36,7 @@ def import_from_webdav_for_all_users(restart=False): retro_count = 0 bgstats_count = 0 ebird_count = 0 + scale_count = 0 for user_id in webdav_enabled_user_ids: client = get_webdav_client(user_id) @@ -43,6 +45,7 @@ def import_from_webdav_for_all_users(restart=False): retro_count += scan_webdav_for_retroarch(client, user_id) bgstats_count += scan_webdav_for_bgstats(client, user_id) ebird_count += scan_webdav_for_ebird(client, user_id) + scale_count += scan_webdav_for_scale(client, user_id) logger.info( "WebDAV import complete", @@ -52,9 +55,10 @@ def import_from_webdav_for_all_users(restart=False): "retroarch": retro_count, "bgstats": bgstats_count, "ebird": ebird_count, + "scale": scale_count, }, ) - return ko_count, gpx_count, retro_count, bgstats_count, ebird_count + return ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count def scan_webdav_for_koreader(webdav_client, user_id, restart=False): @@ -70,9 +74,7 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False): return 0 last_import = ( - KoReaderImport.objects.filter( - user_id=user_id, processed_finished__isnull=False - ) + KoReaderImport.objects.filter(user_id=user_id, processed_finished__isnull=False) .order_by("processed_finished") .last() ) @@ -184,9 +186,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id): try: webdav_client.info(retroarch_path) except: - logger.info( - "No var/retroarch/ directory on webdav", extra={"user_id": user_id} - ) + logger.info("No var/retroarch/ directory on webdav", extra={"user_id": user_id}) return 0 try: @@ -199,9 +199,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id): return 0 lrtl_basenames = sorted( - os.path.basename(fname) - for fname in files - if fname.lower().endswith(".lrtl") + os.path.basename(fname) for fname in files if fname.lower().endswith(".lrtl") ) if not lrtl_basenames: logger.info("No .lrtl files found on webdav", extra={"user_id": user_id}) @@ -229,9 +227,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id): ) downloaded.append(basename) except Exception as e: - logger.error( - "Failed to download retroarch %s: %s", basename, e - ) + logger.error("Failed to download retroarch %s: %s", basename, e) if not downloaded: return 0 @@ -263,7 +259,8 @@ def scan_webdav_for_retroarch(webdav_client, user_id): return 0 zip_path = os.path.join( - download_dir, f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip" + download_dir, + f"retroarch-batch-{datetime.now().strftime('%Y%m%d%H%M%S')}.zip", ) with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: for basename in downloaded: @@ -275,9 +272,7 @@ def scan_webdav_for_retroarch(webdav_client, user_id): files_hash=content_hash, ) with open(zip_path, "rb") as f: - imp.lrtl_file.save( - f"retroarch-batch-{imp.uuid}.zip", f, save=True - ) + imp.lrtl_file.save(f"retroarch-batch-{imp.uuid}.zip", f, save=True) process_retroarch_import.delay(imp.id) logger.info( "Queued retroarch import %s with %d file(s) (hash=%s)", @@ -404,3 +399,62 @@ def scan_webdav_for_ebird(webdav_client, user_id): os.unlink(tmp.name) return new_imports + + +def scan_webdav_for_scale(webdav_client, user_id): + """Download .csv files from WebDAV var/scale/ and queue imports for new files.""" + from scrobbles.models import ScaleCSVImport + from scrobbles.tasks import process_scale_csv_import + + scale_path = ( + DEFAULT_SCALE_PATH # TODO Allow this to be configured in a user profile setting + ) + try: + webdav_client.info(scale_path) + except: + logger.info("No var/scale/ directory on webdav", extra={"user_id": user_id}) + return 0 + + try: + files = webdav_client.list(scale_path) + except Exception as e: + logger.warning( + "Could not list var/scale/", + extra={"user_id": user_id, "error": str(e)}, + ) + return 0 + + new_imports = 0 + already_imported = set( + ScaleCSVImport.objects.filter(user_id=user_id).values_list( + "original_filename", flat=True + ) + ) + + for fname in files: + fname = os.path.basename(fname) + if not fname.lower().endswith(".csv"): + continue + if fname in already_imported: + logger.debug(f"Skipping already-imported {fname}") + continue + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname) + try: + webdav_client.download_sync( + remote_path=f"{scale_path}/{fname}", local_path=tmp.name + ) + imp = ScaleCSVImport.objects.create( + user_id=user_id, + original_filename=fname, + ) + with open(tmp.name, "rb") as f: + imp.csv_file.save(fname, f, save=True) + process_scale_csv_import.delay(imp.id) + new_imports += 1 + except Exception as e: + logger.error(f"Failed to import Scale CSV file {fname}: {e}") + finally: + os.unlink(tmp.name) + + return new_imports diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py index 5b254d9..8705309 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py @@ -14,11 +14,11 @@ class Command(BaseCommand): restart = False if options["restart"]: restart = True - ko_count, gpx_count, retro_count, bgstats_count, ebird_count = ( + ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count = ( webdav.import_from_webdav_for_all_users(restart=restart) ) print( f"Started {ko_count} KOReader, {gpx_count} Trail GPX, " f"{retro_count} Retroarch, {bgstats_count} BGStats, " - f"{ebird_count} eBird WebDAV imports" + f"{ebird_count} eBird, {scale_count} Scale WebDAV imports" ) diff --git a/vrobbler/apps/scrobbles/migrations/0085_scale_csv_import_original_filename.py b/vrobbler/apps/scrobbles/migrations/0085_scale_csv_import_original_filename.py new file mode 100644 index 0000000..3363be8 --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0085_scale_csv_import_original_filename.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.29 on 2026-05-24 15:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scrobbles", "0084_fix_ebird_sequence"), + ] + + operations = [ + migrations.AddField( + model_name="scalecsvimport", + name="original_filename", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index f853d26..ac1a7ef 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -285,6 +285,7 @@ class ScaleCSVImport(BaseFileImportMixin): return path csv_file = models.FileField(upload_to=get_path, **BNULL) + original_filename = models.CharField(max_length=255, **BNULL) def process(self, force=False): from scrobbles.importers.scale import import_scale_csv diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index bee378e..b8b552f 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -168,6 +168,16 @@ def process_ebird_csv_import(import_id): birding_import.process() +@shared_task +def process_scale_csv_import(import_id): + ScaleCSVImport = apps.get_model("scrobbles", "ScaleCSVImport") + scale_import = ScaleCSVImport.objects.filter(id=import_id).first() + if not scale_import: + logger.warn(f"ScaleCSVImport not found with id {import_id}") + return + scale_import.process() + + @shared_task def create_yesterdays_charts(): """Build/update charts for all users starting from last known record.""" diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index 143e1f5..4950a40 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -622,6 +622,9 @@ class ScaleCSVImportCreateView( def form_valid(self, form): self.object = form.save(commit=False) self.object.user = self.request.user + self.object.original_filename = ( + form.cleaned_data["csv_file"].name + ) self.object.save() self.object.process() return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))