From d3146433f2c49553307774e8215e7b6432efdfbb Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Fri, 22 May 2026 11:58:09 -0400 Subject: [PATCH] [tasks] Move retroarch importing to webdav --- vrobbler/apps/scrobbles/importers/webdav.py | 193 +++++++++++++----- .../management/commands/import_from_webdav.py | 9 +- .../0079_retroarchimport_lrtl_file.py | 25 +++ vrobbler/apps/scrobbles/models.py | 43 +++- vrobbler/apps/scrobbles/tasks.py | 14 +- vrobbler/settings.py | 4 - 6 files changed, 219 insertions(+), 69 deletions(-) create mode 100644 vrobbler/apps/scrobbles/migrations/0079_retroarchimport_lrtl_file.py diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index e8e2a07..917e83b 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -13,14 +13,11 @@ logger = logging.getLogger(__name__) DEFAULT_GPX_PATH = "var/gpx/" DEFAULT_KOREADER_PATH = "var/koreader/" +DEFAULT_RETROARCH_PATH = "var/retroarch/" def import_from_webdav_for_all_users(restart=False): - """Grab a list of all users with WebDAV enabled and kickoff imports for them""" - - koreader_path = ( - DEFAULT_KOREADER_PATH + "statistics.sqlite3" - ) # TODO Allow configuring this in user settings + """Iterate all WebDAV-enabled users, scanning each media-type directory.""" webdav_enabled_user_ids = UserProfile.objects.filter( webdav_url__isnull=False, webdav_user__isnull=False, @@ -29,66 +26,87 @@ def import_from_webdav_for_all_users(restart=False): ).values_list("user_id", flat=True) logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts") - koreader_import_count = 0 - trail_gpx_import_count = 0 + ko_count = 0 + gpx_count = 0 + retro_count = 0 for user_id in webdav_enabled_user_ids: - webdav_client = get_webdav_client(user_id) + client = get_webdav_client(user_id) + ko_count += scan_webdav_for_koreader(client, user_id, restart) + gpx_count += scan_webdav_for_gpx(client, user_id) + retro_count += scan_webdav_for_retroarch(client, user_id) - try: - webdav_client.info(koreader_path) - koreader_found = True - except: - koreader_found = False - logger.info( - "No koreader stats file found on webdav", - extra={"user_id": user_id}, - ) + logger.info( + "WebDAV import complete", + extra={ + "koreader": ko_count, + "trail_gpx": gpx_count, + "retroarch": retro_count, + }, + ) + return ko_count, gpx_count, retro_count - trail_gpx_import_count += scan_webdav_for_gpx(webdav_client, user_id) - if koreader_found: - last_import = ( - KoReaderImport.objects.filter( - user_id=user_id, processed_finished__isnull=False - ) - .order_by("processed_finished") - .last() - ) +def scan_webdav_for_koreader(webdav_client, user_id, restart=False): + """Check for koreader statistics.sqlite3 and queue import if changed.""" + koreader_path = DEFAULT_KOREADER_PATH + "statistics.sqlite3" + try: + webdav_client.info(koreader_path) + except: + logger.info( + "No koreader stats file found on webdav", + extra={"user_id": user_id}, + ) + return 0 - new_hash = get_file_md5_hash(fetch_file_from_webdav(1)) - old_hash = None - if last_import: - old_hash = last_import.file_md5_hash() + last_import = ( + KoReaderImport.objects.filter( + user_id=user_id, processed_finished__isnull=False + ) + .order_by("processed_finished") + .last() + ) - if old_hash and new_hash == old_hash: - logger.info( - "koreader stats file has not changed", - extra={ - "user_id": user_id, - "new_hash": new_hash, - "old_hash": old_hash, - "last_import_id": last_import.id, - }, - ) - continue + file_path = fetch_file_from_webdav(user_id) + if not file_path: + logger.warning( + "Could not fetch koreader file from webdav", + extra={"user_id": user_id}, + ) + return 0 - koreader_import, created = KoReaderImport.objects.get_or_create( - user_id=user_id, processed_finished__isnull=True - ) + new_hash = get_file_md5_hash(file_path) + old_hash = last_import.file_md5_hash() if last_import else None - if not created and not restart: - logger.info( - f"Not resuming failed KoReader import {koreader_import.id} for user {user_id}, use restart=True to restart" - ) - continue + if last_import and new_hash == old_hash: + logger.info( + "koreader stats file has not changed", + extra={ + "user_id": user_id, + "new_hash": new_hash, + "old_hash": old_hash, + "last_import_id": last_import.id, + }, + ) + os.unlink(file_path) + return 0 - koreader_import.save_sqlite_file_to_self(koreader_file_path) + koreader_import, created = KoReaderImport.objects.get_or_create( + user_id=user_id, processed_finished__isnull=True + ) - process_koreader_import.delay(koreader_import.id) - koreader_import_count += 1 + if not created and not restart: + logger.info( + f"Not resuming failed KoReader import {koreader_import.id} " + f"for user {user_id}, use restart=True to restart" + ) + os.unlink(file_path) + return 0 - return koreader_import_count, trail_gpx_import_count + koreader_import.save_sqlite_file_to_self(file_path) + process_koreader_import.delay(koreader_import.id) + os.unlink(file_path) + return 1 def scan_webdav_for_gpx(webdav_client, user_id): @@ -142,3 +160,72 @@ def scan_webdav_for_gpx(webdav_client, user_id): os.unlink(tmp.name) return new_imports + + +def scan_webdav_for_retroarch(webdav_client, user_id): + """Download .lrtl files from WebDAV and import retroarch playlog data.""" + retroarch_path = DEFAULT_RETROARCH_PATH + try: + webdav_client.info(retroarch_path) + except: + logger.info( + "No var/retroarch/ directory on webdav", extra={"user_id": user_id} + ) + return 0 + + try: + files = webdav_client.list(retroarch_path) + except Exception as e: + logger.warning( + "Could not list var/retroarch/", + extra={"user_id": user_id, "error": str(e)}, + ) + return 0 + + lrtl_files = [ + fname + for fname in files + if os.path.basename(fname).lower().endswith(".lrtl") + ] + if not lrtl_files: + logger.info( + "No .lrtl files found on webdav", extra={"user_id": user_id} + ) + return 0 + + from scrobbles.models import RetroarchImport + from scrobbles.tasks import process_retroarch_import + + already_imported = set( + RetroarchImport.objects.filter(user_id=user_id).values_list( + "original_filename", flat=True + ) + ) + + new_imports = 0 + for fname in lrtl_files: + basename = os.path.basename(fname) + if basename in already_imported: + logger.debug(f"Skipping already-imported {basename}") + continue + + tmp = tempfile.NamedTemporaryFile(delete=False, suffix=basename) + try: + webdav_client.download_sync( + remote_path=f"{retroarch_path}/{basename}", + local_path=tmp.name, + ) + imp = RetroarchImport.objects.create( + user_id=user_id, + original_filename=basename, + ) + with open(tmp.name, "rb") as f: + imp.lrtl_file.save(basename, f, save=True) + process_retroarch_import.delay(imp.id) + new_imports += 1 + except Exception as e: + logger.error(f"Failed to import retroarch {basename}: {e}") + finally: + os.unlink(tmp.name) + + return new_imports diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py index 7bb480c..fc78d21 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py @@ -14,5 +14,10 @@ class Command(BaseCommand): restart = False if options["restart"]: restart = True - ko_count, gpx_count = webdav.import_from_webdav_for_all_users(restart=restart) - print(f"Started {ko_count} KOReader and {gpx_count} Trail GPX WebDAV imports") + ko_count, gpx_count, retro_count = webdav.import_from_webdav_for_all_users( + restart=restart + ) + print( + f"Started {ko_count} KOReader, {gpx_count} Trail GPX, " + f"and {retro_count} Retroarch WebDAV imports" + ) diff --git a/vrobbler/apps/scrobbles/migrations/0079_retroarchimport_lrtl_file.py b/vrobbler/apps/scrobbles/migrations/0079_retroarchimport_lrtl_file.py new file mode 100644 index 0000000..4b82c57 --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0079_retroarchimport_lrtl_file.py @@ -0,0 +1,25 @@ +# Generated by Django 4.2.29 on 2026-05-22 15:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("scrobbles", "0078_trailgpximport"), + ] + + operations = [ + migrations.AddField( + model_name="retroarchimport", + name="lrtl_file", + field=models.FileField( + blank=True, null=True, upload_to="scrobbles/lrtl_file/" + ), + ), + migrations.AddField( + model_name="retroarchimport", + name="original_filename", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index b78b308..e7c21c7 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -399,6 +399,9 @@ class RetroarchImport(BaseFileImportMixin): def get_absolute_url(self): return reverse("scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid}) + original_filename = models.CharField(max_length=255, **BNULL) + lrtl_file = models.FileField(upload_to="scrobbles/lrtl_file/", **BNULL) + def process(self, import_all=False, force=False): """Import scrobbles found on Retroarch""" if self.user.id == 1: @@ -411,17 +414,39 @@ class RetroarchImport(BaseFileImportMixin): if force: logger.info(f"You told me to force import from Retroarch") - if not self.user.profile.retroarch_path: - logger.info( - "Tying to import Retroarch logs, but user has no retroarch_path configured" - ) - self.mark_started() - scrobbles = retroarch.import_retroarch_lrtl_files( - self.user.profile.retroarch_path, - self.user.id, - ) + if self.lrtl_file: + import tempfile + import os + + tmpdir = tempfile.mkdtemp() + try: + src_path = self.lrtl_file.path + basename = self.original_filename or os.path.basename(src_path) + dst = os.path.join(tmpdir, basename) + with open(dst, "wb") as f: + f.write(self.lrtl_file.read()) + scrobbles = retroarch.import_retroarch_lrtl_files( + tmpdir + "/", + self.user.id, + ) + finally: + import shutil + + shutil.rmtree(tmpdir, ignore_errors=True) + else: + if not self.user.profile.retroarch_path: + logger.info( + "Tying to import Retroarch logs, but user has no retroarch_path configured" + ) + self.mark_finished() + return + + scrobbles = retroarch.import_retroarch_lrtl_files( + self.user.profile.retroarch_path, + self.user.id, + ) self.record_log(scrobbles) self.mark_finished() diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index 29924d0..636975d 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -387,7 +387,19 @@ def import_from_lastfm_all_users(): @shared_task def import_from_retroarch_all_users(): - """Import RetroArch scrobbles for all users (replaces @daily cron).""" + """Import RetroArch scrobbles for all users (replaces @daily cron). + + Deprecated: retroarch .lrtl files are now picked up by the WebDAV + importer (scan_webdav_for_retroarch). This task remains for manual use. + """ + import warnings + + warnings.warn( + "import_from_retroarch_all_users is deprecated. " + "Upload .lrtl files to WebDAV var/retroarch/ instead.", + DeprecationWarning, + stacklevel=2, + ) from scrobbles.utils import import_retroarch_for_all_users import_retroarch_for_all_users() diff --git a/vrobbler/settings.py b/vrobbler/settings.py index 247a235..1d25754 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -135,10 +135,6 @@ CELERY_BEAT_SCHEDULE = { "task": "scrobbles.tasks.import_from_lastfm_all_users", "schedule": crontab(minute="*/30"), }, - "import-from-retroarch": { - "task": "scrobbles.tasks.import_from_retroarch_all_users", - "schedule": crontab(hour=0, minute=0), - }, "import-from-webdav": { "task": "scrobbles.tasks.import_from_webdav_all_users", "schedule": crontab(minute="*/2"),