From dd2f44e72fe4da96eafef6d4620d7a0f1d6fd810 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Fri, 22 May 2026 12:40:48 -0400 Subject: [PATCH] [boardgames] Move importer from IMAP to WebDAV --- vrobbler/apps/scrobbles/importers/webdav.py | 65 ++++++++++++++++++- .../management/commands/import_from_webdav.py | 6 +- vrobbler/apps/scrobbles/tasks.py | 11 +++- vrobbler/settings.py | 9 +-- 4 files changed, 80 insertions(+), 11 deletions(-) diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index 95e811c..fab4246 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -2,9 +2,12 @@ import logging import os import tempfile +import json + from books.koreader import fetch_file_from_webdav from profiles.models import UserProfile from scrobbles.models import KoReaderImport, TrailGPXImport +from scrobbles.scrobblers import email_scrobble_board_game from scrobbles.tasks import process_koreader_import, process_trail_gpx_import from scrobbles.utils import get_file_md5_hash from webdav.client import get_webdav_client @@ -14,6 +17,7 @@ logger = logging.getLogger(__name__) DEFAULT_GPX_PATH = "var/gpx/" DEFAULT_KOREADER_PATH = "var/koreader/" DEFAULT_RETROARCH_PATH = "var/retroarch/" +DEFAULT_BGSTATS_PATH = "var/bgstats/" def import_from_webdav_for_all_users(restart=False): @@ -29,12 +33,14 @@ def import_from_webdav_for_all_users(restart=False): ko_count = 0 gpx_count = 0 retro_count = 0 + bgstats_count = 0 for user_id in webdav_enabled_user_ids: client = get_webdav_client(user_id) ko_count += scan_webdav_for_koreader(client, user_id, restart) gpx_count += scan_webdav_for_gpx(client, user_id) retro_count += scan_webdav_for_retroarch(client, user_id) + bgstats_count += scan_webdav_for_bgstats(client, user_id) logger.info( "WebDAV import complete", @@ -42,9 +48,10 @@ def import_from_webdav_for_all_users(restart=False): "koreader": ko_count, "trail_gpx": gpx_count, "retroarch": retro_count, + "bgstats": bgstats_count, }, ) - return ko_count, gpx_count, retro_count + return ko_count, gpx_count, retro_count, bgstats_count def scan_webdav_for_koreader(webdav_client, user_id, restart=False): @@ -280,3 +287,59 @@ def scan_webdav_for_retroarch(webdav_client, user_id): import shutil shutil.rmtree(download_dir, ignore_errors=True) + + +def scan_webdav_for_bgstats(webdav_client, user_id): + """Download .bgsplay files from WebDAV, parse JSON, and scrobble board games.""" + bgstats_path = DEFAULT_BGSTATS_PATH + try: + webdav_client.info(bgstats_path) + except: + logger.info( + "No var/bgstats/ directory on webdav", extra={"user_id": user_id} + ) + return 0 + + try: + files = webdav_client.list(bgstats_path) + except Exception as e: + logger.warning( + "Could not list var/bgstats/", + extra={"user_id": user_id, "error": str(e)}, + ) + return 0 + + processed = 0 + seen_this_cycle = set() + + for fname in files: + fname = os.path.basename(fname) + if not fname.lower().endswith(".bgsplay"): + continue + if fname in seen_this_cycle: + continue + seen_this_cycle.add(fname) + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname) + try: + webdav_client.download_sync( + remote_path=f"{bgstats_path}/{fname}", local_path=tmp.name + ) + with open(tmp.name, "r", encoding="utf-8") as f: + parsed_json = json.load(f) + scrobbles = email_scrobble_board_game(parsed_json, user_id) + logger.info( + "BG Stats import from %s created %d scrobble(s)", + fname, + len(scrobbles), + extra={"user_id": user_id, "filename": fname}, + ) + processed += 1 + except Exception as e: + logger.error( + "Failed to import BG Stats file %s: %s", fname, e + ) + finally: + os.unlink(tmp.name) + + return processed \ No newline at end of file diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py index fc78d21..bd176bc 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py @@ -14,10 +14,10 @@ class Command(BaseCommand): restart = False if options["restart"]: restart = True - ko_count, gpx_count, retro_count = webdav.import_from_webdav_for_all_users( - restart=restart + ko_count, gpx_count, retro_count, bgstats_count = ( + webdav.import_from_webdav_for_all_users(restart=restart) ) print( f"Started {ko_count} KOReader, {gpx_count} Trail GPX, " - f"and {retro_count} Retroarch WebDAV imports" + f"{retro_count} Retroarch, {bgstats_count} BGStats WebDAV imports" ) diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index f98dcd0..edd2982 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -415,10 +415,15 @@ def import_from_webdav_all_users(): @shared_task def import_from_imap_all_users(): - """Import from IMAP for all users (replaces */4 cron).""" - from vrobbler.apps.scrobbles.importers.imap import import_scrobbles_from_imap + """Deprecated — BG Stats files now picked up from WebDAV var/bgstats/.""" + import warnings - import_scrobbles_from_imap() + warnings.warn( + "IMAP import is deprecated. Upload .bgsplay files to WebDAV var/bgstats/ instead.", + DeprecationWarning, + stacklevel=2, + ) + logger.warning("Skipping deprecated IMAP import (use WebDAV var/bgstats/ instead)") @shared_task diff --git a/vrobbler/settings.py b/vrobbler/settings.py index a68645b..7d3f613 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -140,10 +140,11 @@ CELERY_BEAT_SCHEDULE = { "task": "scrobbles.tasks.import_from_webdav_all_users", "schedule": crontab(minute="*/2"), }, - "import-from-imap": { - "task": "scrobbles.tasks.import_from_imap_all_users", - "schedule": crontab(minute="*/4"), - }, + # Deprecated: BG Stats files now picked up from WebDAV var/bgstats/ + # "import-from-imap": { + # "task": "scrobbles.tasks.import_from_imap_all_users", + # "schedule": crontab(minute="*/4"), + # }, "import-from-lichess": { "task": "scrobbles.tasks.import_from_lichess_all_users", "schedule": crontab(minute="*/20"),