134 lines
4.4 KiB
Python
134 lines
4.4 KiB
Python
import logging
|
|
import os
|
|
import tempfile
|
|
|
|
from books.koreader import fetch_file_from_webdav
|
|
from profiles.models import UserProfile
|
|
from scrobbles.models import KoReaderImport, TrailGPXImport
|
|
from scrobbles.tasks import process_koreader_import, process_trail_gpx_import
|
|
from scrobbles.utils import get_file_md5_hash
|
|
from webdav.client import get_webdav_client
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def import_from_webdav_for_all_users(restart=False):
|
|
"""Grab a list of all users with WebDAV enabled and kickoff imports for them"""
|
|
|
|
webdav_enabled_user_ids = UserProfile.objects.filter(
|
|
webdav_url__isnull=False,
|
|
webdav_user__isnull=False,
|
|
webdav_pass__isnull=False,
|
|
webdav_auto_import=True,
|
|
).values_list("user_id", flat=True)
|
|
logger.info(f"Start import of {webdav_enabled_user_ids.count()} webdav accounts")
|
|
|
|
koreader_import_count = 0
|
|
trail_gpx_import_count = 0
|
|
|
|
for user_id in webdav_enabled_user_ids:
|
|
webdav_client = get_webdav_client(user_id)
|
|
|
|
try:
|
|
webdav_client.info("var/koreader/statistics.sqlite3")
|
|
koreader_found = True
|
|
except:
|
|
koreader_found = False
|
|
logger.info(
|
|
"No koreader stats file found on webdav",
|
|
extra={"user_id": user_id},
|
|
)
|
|
|
|
if koreader_found:
|
|
last_import = (
|
|
KoReaderImport.objects.filter(
|
|
user_id=user_id, processed_finished__isnull=False
|
|
)
|
|
.order_by("processed_finished")
|
|
.last()
|
|
)
|
|
|
|
koreader_file_path = fetch_file_from_webdav(1)
|
|
new_hash = get_file_md5_hash(koreader_file_path)
|
|
old_hash = None
|
|
if last_import:
|
|
old_hash = last_import.file_md5_hash()
|
|
|
|
if old_hash and new_hash == old_hash:
|
|
logger.info(
|
|
"koreader stats file has not changed",
|
|
extra={
|
|
"user_id": user_id,
|
|
"new_hash": new_hash,
|
|
"old_hash": old_hash,
|
|
"last_import_id": last_import.id,
|
|
},
|
|
)
|
|
continue
|
|
|
|
koreader_import, created = KoReaderImport.objects.get_or_create(
|
|
user_id=user_id, processed_finished__isnull=True
|
|
)
|
|
|
|
if not created and not restart:
|
|
logger.info(
|
|
f"Not resuming failed KoReader import {koreader_import.id} for user {user_id}, use restart=True to restart"
|
|
)
|
|
continue
|
|
|
|
koreader_import.save_sqlite_file_to_self(koreader_file_path)
|
|
|
|
process_koreader_import.delay(koreader_import.id)
|
|
koreader_import_count += 1
|
|
|
|
trail_gpx_import_count += scan_webdav_for_gpx(webdav_client, user_id)
|
|
|
|
return koreader_import_count, trail_gpx_import_count
|
|
|
|
|
|
def scan_webdav_for_gpx(webdav_client, user_id):
|
|
try:
|
|
webdav_client.info("gpx/")
|
|
except:
|
|
logger.info("No gpx/ directory on webdav", extra={"user_id": user_id})
|
|
return 0
|
|
|
|
try:
|
|
files = webdav_client.list("gpx/")
|
|
except Exception as e:
|
|
logger.warning("Could not list gpx/", extra={"user_id": user_id, "error": str(e)})
|
|
return 0
|
|
|
|
gpx_extensions = (".gpx", ".fit")
|
|
new_imports = 0
|
|
already_imported = set(
|
|
TrailGPXImport.objects.filter(user_id=user_id)
|
|
.values_list("original_filename", flat=True)
|
|
)
|
|
|
|
for fname in files:
|
|
fname = os.path.basename(fname)
|
|
if not fname.lower().endswith(gpx_extensions):
|
|
continue
|
|
if fname in already_imported:
|
|
logger.debug(f"Skipping already-imported {fname}")
|
|
continue
|
|
|
|
tmp = tempfile.NamedTemporaryFile(delete=False, suffix=fname)
|
|
try:
|
|
webdav_client.download_sync(remote_path=f"gpx/{fname}", local_path=tmp.name)
|
|
imp = TrailGPXImport.objects.create(
|
|
user_id=user_id,
|
|
original_filename=fname,
|
|
)
|
|
with open(tmp.name, "rb") as f:
|
|
imp.gpx_file.save(fname, f, save=True)
|
|
process_trail_gpx_import.delay(imp.id)
|
|
new_imports += 1
|
|
except Exception as e:
|
|
logger.error(f"Failed to import {fname}: {e}")
|
|
finally:
|
|
os.unlink(tmp.name)
|
|
|
|
return new_imports
|