diff --git a/vrobbler/apps/scrobbles/importers/__init__.py b/vrobbler/apps/scrobbles/importers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/scrobbles/imap.py b/vrobbler/apps/scrobbles/importers/imap.py similarity index 98% rename from vrobbler/apps/scrobbles/imap.py rename to vrobbler/apps/scrobbles/importers/imap.py index 7b68789..59a8e49 100644 --- a/vrobbler/apps/scrobbles/imap.py +++ b/vrobbler/apps/scrobbles/importers/imap.py @@ -1,16 +1,17 @@ -import json -import imaplib import email +import imaplib +import json +import logging from email.header import decode_header + from profiles.models import UserProfile from scrobbles.models import Scrobble from scrobbles.scrobblers import email_scrobble_board_game -import logging logger = logging.getLogger(__name__) -def process_scrobbles_from_imap() -> list[Scrobble]: +def import_scrobbles_from_imap() -> list[Scrobble]: """For all user profiles with IMAP creds, check inbox for scrobbleable email attachments.""" scrobbles_created: list[Scrobble] = [] diff --git a/vrobbler/apps/scrobbles/tsv.py b/vrobbler/apps/scrobbles/importers/tsv.py similarity index 100% rename from vrobbler/apps/scrobbles/tsv.py rename to vrobbler/apps/scrobbles/importers/tsv.py diff --git a/vrobbler/apps/scrobbles/importers/webdav.py b/vrobbler/apps/scrobbles/importers/webdav.py new file mode 100644 index 0000000..42e9a15 --- /dev/null +++ b/vrobbler/apps/scrobbles/importers/webdav.py @@ -0,0 +1,81 @@ +from books.koreader import fetch_file_from_webdav +from profiles.models import UserProfile +from scrobbles.models import KoReaderImport +from scrobbles.tasks import process_koreader_import +from webdav.client import get_webdav_client +import logging + +logger = logging.getLogger(__name__) + + +def import_from_webdav_for_all_users(restart=False): + """Grab a list of all users with WebDAV enabled and kickoff imports for them""" + + # LastFmImport = apps.get_model("scrobbles", "LastFMImport") + webdav_enabled_user_ids = UserProfile.objects.filter( + webdav_url__isnull=False, + webdav_user__isnull=False, + webdav_pass__isnull=False, + webdav_auto_import=True, + ).values_list("user_id", flat=True) + logger.info( + f"Start import of {webdav_enabled_user_ids.count()} webdav accounts" + ) + + koreader_import_count = 0 + + for user_id in webdav_enabled_user_ids: + webdav_client = get_webdav_client(user_id) + + try: + webdav_client.info("var/koreader/statistics.sqlite3") + koreader_found = True + except: + koreader_found = False + logger.info( + "No koreader stats file found on webdav", + extra={"user_id": user_id}, + ) + + if koreader_found: + last_import = ( + KoReaderImport.objects.filter( + user_id=user_id, processed_finished__isnull=False + ) + .order_by("Processed_finished") + .last() + ) + + koreader_file_path = fetch_file_from_webdav(1) + new_hash = get_file_md5_hash(koreader_file_path) + old_hash = None + if last_import: + old_hash = last_import.file_md5_hash() + + if old_hash and new_hash == old_hash: + logger.info( + "koreader stats file has not changed", + extra={ + "user_id": user_id, + "new_hash": new_hash, + "old_hash": old_hash, + "last_import_id": last_import.id, + }, + ) + continue + + koreader_import, created = KoReaderImport.objects.get_or_create( + user_id=user_id, processed_finished__isnull=True + ) + + if not created and not restart: + logger.info( + f"Not resuming failed KoReader import {koreader_import.id} for user {user_id}, use restart=True to restart" + ) + continue + + koreader_import.save_sqlite_file_to_self(koreader_file_path) + + process_koreader_import.delay(koreader_import.id) + koreader_import_count += 1 + return koreader_import_count diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_imap.py b/vrobbler/apps/scrobbles/management/commands/import_from_imap.py index 7da97e4..96271bc 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_imap.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_imap.py @@ -1,5 +1,5 @@ from django.core.management.base import BaseCommand -from vrobbler.apps.scrobbles.imap import process_scrobbles_from_imap +from vrobbler.apps.scrobbles.importers.imap import import_scrobbles_from_imap class Command(BaseCommand): @@ -11,5 +11,5 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - count = len(process_scrobbles_from_imap()) + count = len(import_scrobbles_from_imap()) print(f"Started {count} IMAP imports") diff --git a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py index 707d056..b9c2460 100644 --- a/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py +++ b/vrobbler/apps/scrobbles/management/commands/import_from_webdav.py @@ -1,5 +1,5 @@ from django.core.management.base import BaseCommand -from vrobbler.apps.scrobbles.utils import import_from_webdav_for_all_users +from vrobbler.apps.scrobbles.importers import webdav class Command(BaseCommand): @@ -14,5 +14,5 @@ class Command(BaseCommand): restart = False if options["restart"]: restart = True - count = import_from_webdav_for_all_users(restart=restart) + count = webdav.import_from_webdav_for_all_users(restart=restart) print(f"Started {count} WeDAV imports")