From 20c78744660cee23f397778541f038869f602a6a Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Thu, 28 May 2026 17:22:56 -0400 Subject: [PATCH] [importers] Add processed directory flow to board games --- PROJECT.org | 13 ++++- vrobbler/apps/scrobbles/importers/webdav.py | 63 +++++++++++++++++++-- 2 files changed, 70 insertions(+), 6 deletions(-) diff --git a/PROJECT.org b/PROJECT.org index d25dc49..667683f 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -93,7 +93,7 @@ fetching and simple saving. :LOGBOOK: CLOCK: [2025-07-09 Wed 09:55]--[2025-07-09 Wed 10:15] => 0:20 :END: -* Backlog [28/44] :vrobbler:project:personal: +* Backlog [30/47] :vrobbler:project:personal: ** TODO [#C] Add sentiment parsing for Scrobbles with notes :vrobbler:project:scrobbles:sentiment: :PROPERTIES: :ID: 37781d6a-f3b0-48b2-bf98-33c2c791cf85 @@ -504,7 +504,16 @@ Billy` - Same pattern as the GPX importer: after importing a =.csv= file from WebDAV, move it to =var/ebird/processed/= with a timestamp appended. -** TODO [#A] Move imported Scale CSV files to processed/ directory on WebDAV :webdav:scale:importers: +** DONE [#A] Move imported Board Game CSV files to processed/ directory on WebDAV :webdav:boardgames:importers: +:PROPERTIES: +:ID: a3f8d30c-d2c3-4ee7-b062-f0f16bd9b0b4 +:END: + + - File: ~vrobbler/apps/scrobbles/importers/webdav.py~ (line 496) + - Same pattern as the GPX importer: after importing a =.csv= file from + WebDAV, move it to =var/bgstats/processed/= with a timestamp appended. + +** DONE [#A] Move imported Scale CSV files to processed/ directory on WebDAV :webdav:scale:importers: :PROPERTIES: :ID: 1a0de363-d1ea-466e-9966-e24941a6180b :END: diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index 8b68773..b0e64ac 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -66,7 +66,9 @@ def import_from_webdav_for_all_users( client, user_id, update_hash_only=update_retroarch_hash ) logger.info("Scanning WebDAV bgstats for user %s", user_id) - bgstats_count += scan_webdav_for_bgstats(client, user_id) + bgstats_count += scan_webdav_for_bgstats( + client, user_id, include_processed=include_processed + ) logger.info("Scanning WebDAV ebird for user %s", user_id) ebird_count += scan_webdav_for_ebird(client, user_id) logger.info("Scanning WebDAV scale for user %s", user_id) @@ -438,8 +440,12 @@ def scan_webdav_for_retroarch(webdav_client, user_id, update_hash_only=False): shutil.rmtree(download_dir, ignore_errors=True) -def scan_webdav_for_bgstats(webdav_client, user_id): - """Download .bgsplay files from WebDAV and queue imports for new files.""" +def scan_webdav_for_bgstats(webdav_client, user_id, include_processed=False): + """Download .bgsplay files from WebDAV and queue imports for new files. + + After importing, files are moved to var/bgstats/processed/ so they are + not re-imported on subsequent scans unless *include_processed* is True. + """ from scrobbles.models import BGStatsImport from scrobbles.tasks import process_bgstats_import @@ -459,6 +465,12 @@ def scan_webdav_for_bgstats(webdav_client, user_id): ) return 0 + processed_dir = f"{bgstats_path}processed/" + try: + webdav_client.mkdir(processed_dir, recursive=True) + except Exception: + pass + new_imports = 0 already_imported = set( BGStatsImport.objects.filter(user_id=user_id).values_list( @@ -470,6 +482,8 @@ def scan_webdav_for_bgstats(webdav_client, user_id): fname = os.path.basename(fname) if not fname.lower().endswith(".bgsplay"): continue + if fname == "processed": + continue if fname in already_imported: logger.debug(f"Skipping already-imported {fname}") continue @@ -477,7 +491,7 @@ def scan_webdav_for_bgstats(webdav_client, user_id): tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname) try: webdav_client.download_sync( - remote_path=f"{bgstats_path}/{fname}", local_path=tmp.name + remote_path=f"{bgstats_path}{fname}", local_path=tmp.name ) imp = BGStatsImport.objects.create( user_id=user_id, @@ -485,6 +499,14 @@ def scan_webdav_for_bgstats(webdav_client, user_id): ) with open(tmp.name, "rb") as f: imp.bgsplay_file.save(fname, f, save=True) + + stem, ext = os.path.splitext(fname) + ts = datetime.now().strftime("%Y%m%dT%H%M%S") + webdav_client.move( + f"{bgstats_path}{fname}", + f"{processed_dir}{stem}_{ts}{ext}", + ) + process_bgstats_import.delay(imp.id) new_imports += 1 except Exception as e: @@ -492,6 +514,39 @@ def scan_webdav_for_bgstats(webdav_client, user_id): finally: os.unlink(tmp.name) + if include_processed: + try: + processed_files = webdav_client.list(processed_dir) + except Exception as e: + logger.warning( + "Could not list var/bgstats/processed/", + extra={"user_id": user_id, "error": str(e)}, + ) + return new_imports + + for fname in processed_files: + fname = os.path.basename(fname) + if not fname.lower().endswith(".bgsplay"): + continue + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname) + try: + webdav_client.download_sync( + remote_path=f"{processed_dir}{fname}", local_path=tmp.name + ) + imp = BGStatsImport.objects.create( + user_id=user_id, + original_filename=fname, + ) + with open(tmp.name, "rb") as f: + imp.bgsplay_file.save(fname, f, save=True) + process_bgstats_import.delay(imp.id) + new_imports += 1 + except Exception as e: + logger.error(f"Failed to import processed BG Stats file {fname}: {e}") + finally: + os.unlink(tmp.name) + return new_imports