diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py index f7ee58c..8d06044 100644 --- a/vrobbler/apps/scrobbles/importers/webdav.py +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -2,6 +2,7 @@ import json import logging import os import tempfile +from datetime import datetime from books.koreader import fetch_file_from_webdav from profiles.models import UserProfile @@ -20,7 +21,9 @@ DEFAULT_EBIRD_PATH = "var/ebird/" DEFAULT_SCALE_PATH = "var/scale/" -def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False, update_koreader_etag=False): +def import_from_webdav_for_all_users( + restart=False, update_retroarch_hash=False, update_koreader_etag=False +): """Iterate all WebDAV-enabled users, scanning each media-type directory.""" webdav_enabled_user_ids = UserProfile.objects.filter( webdav_url__isnull=False, @@ -78,7 +81,9 @@ def import_from_webdav_for_all_users(restart=False, update_retroarch_hash=False, return ko_count, gpx_count, retro_count, bgstats_count, ebird_count, scale_count -def scan_webdav_for_koreader(webdav_client, user_id, restart=False, update_etag_only=False): +def scan_webdav_for_koreader( + webdav_client, user_id, restart=False, update_etag_only=False +): """Check for koreader statistics.sqlite3 and queue import if changed. Uses the remote file's ETag (from a single PROPFIND) to detect changes @@ -110,7 +115,7 @@ def scan_webdav_for_koreader(webdav_client, user_id, restart=False, update_etag_ sqlite_info = None for f in files: - if (f.get("name") or "") == "statistics.sqlite3": + if "statistics.sqlite3" in f.get("path"): sqlite_info = f break @@ -213,25 +218,25 @@ def scan_webdav_for_gpx(webdav_client, user_id): return 0 gpx_extensions = (".gpx", ".fit") + processed_dir = f"{gpx_path}processed/" + try: + webdav_client.mkdir(processed_dir, recursive=True) + except Exception: + pass + new_imports = 0 - already_imported = set( - TrailGPXImport.objects.filter(user_id=user_id).values_list( - "original_filename", flat=True - ) - ) for fname in files: fname = os.path.basename(fname) if not fname.lower().endswith(gpx_extensions): continue - if fname in already_imported: - logger.debug(f"Skipping already-imported {fname}") + if fname == "processed": continue tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname) try: webdav_client.download_sync( - remote_path=f"{gpx_path}/{fname}", local_path=tmp.name + remote_path=f"{gpx_path}{fname}", local_path=tmp.name ) imp = TrailGPXImport.objects.create( user_id=user_id, @@ -239,6 +244,14 @@ def scan_webdav_for_gpx(webdav_client, user_id): ) with open(tmp.name, "rb") as f: imp.gpx_file.save(fname, f, save=True) + + stem, ext = os.path.splitext(fname) + ts = datetime.now().strftime("%Y%m%dT%H%M%S") + webdav_client.move( + f"{gpx_path}{fname}", + f"{processed_dir}{stem}_{ts}{ext}", + ) + process_trail_gpx_import.delay(imp.id) new_imports += 1 except Exception as e: