[importers] Reorganize importers a little
This commit is contained in:
0
vrobbler/apps/scrobbles/importers/__init__.py
Normal file
0
vrobbler/apps/scrobbles/importers/__init__.py
Normal file
107
vrobbler/apps/scrobbles/importers/imap.py
Normal file
107
vrobbler/apps/scrobbles/importers/imap.py
Normal file
@ -0,0 +1,107 @@
|
||||
import email
|
||||
import imaplib
|
||||
import json
|
||||
import logging
|
||||
from email.header import decode_header
|
||||
|
||||
from profiles.models import UserProfile
|
||||
from scrobbles.models import Scrobble
|
||||
from scrobbles.scrobblers import email_scrobble_board_game
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def import_scrobbles_from_imap() -> list[Scrobble]:
|
||||
"""For all user profiles with IMAP creds, check inbox for scrobbleable email attachments."""
|
||||
scrobbles_created: list[Scrobble] = []
|
||||
|
||||
active_profiles = UserProfile.objects.filter(imap_auto_import=True)
|
||||
logger.info(
|
||||
"Starting import of scrobbles from IMAP",
|
||||
extra={"active_profiles": active_profiles},
|
||||
)
|
||||
for profile in active_profiles:
|
||||
logger.info(
|
||||
"Importing scrobbles from IMAP for user",
|
||||
extra={"user_id": profile.user_id},
|
||||
)
|
||||
mail = imaplib.IMAP4_SSL(profile.imap_url)
|
||||
mail.login(profile.imap_user, profile.imap_pass)
|
||||
mail.select("INBOX") # TODO configure this in profile
|
||||
|
||||
# Search for unseen emails
|
||||
status, messages = mail.search(None, "UnSeen")
|
||||
if status != "OK":
|
||||
logger.info("IMAP status not OK", extra={"status": status})
|
||||
return
|
||||
|
||||
for uid in messages[0].split():
|
||||
status, msg_data = mail.fetch(uid, "(RFC822)")
|
||||
if status != "OK":
|
||||
logger.info("IMAP status not OK", extra={"status": status})
|
||||
continue
|
||||
|
||||
try:
|
||||
message = email.message_from_bytes(msg_data[0][1])
|
||||
logger.info(
|
||||
"Processing email message", extra={"email_msg": message}
|
||||
)
|
||||
except IndexError:
|
||||
logger.info("No email message data found")
|
||||
return
|
||||
|
||||
# Decode subject safely
|
||||
subject, encoding = decode_header(message["Subject"])[0]
|
||||
if isinstance(subject, bytes):
|
||||
subject = subject.decode(encoding or "utf-8")
|
||||
if "live activity now!" in subject:
|
||||
print("Found a Garmin email")
|
||||
|
||||
for part in message.walk():
|
||||
if part.get_content_disposition() == "attachment":
|
||||
filename = part.get_filename()
|
||||
if filename:
|
||||
# Decode the filename if necessary
|
||||
decoded_name, encoding = decode_header(filename)[0]
|
||||
if isinstance(decoded_name, bytes):
|
||||
filename = decoded_name.decode(encoding or "utf-8")
|
||||
|
||||
file_data = part.get_payload(decode=True)
|
||||
|
||||
parsed_json = ""
|
||||
|
||||
# Try parsing JSON if applicable
|
||||
parsed_json = {}
|
||||
if filename.lower().endswith(".bgsplay"):
|
||||
# TODO Pull this out into a parse_pgsplay function
|
||||
try:
|
||||
parsed_json = json.loads(
|
||||
file_data.decode("utf-8")
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"Failed to parse JSON file",
|
||||
extra={"filename": filename, "error": e},
|
||||
)
|
||||
|
||||
if not parsed_json:
|
||||
logger.info(
|
||||
"No JSON found in BG Stats file",
|
||||
extra={"filename": filename},
|
||||
)
|
||||
continue
|
||||
|
||||
scrobbles_created = email_scrobble_board_game(
|
||||
parsed_json, profile.user_id
|
||||
)
|
||||
|
||||
mail.logout()
|
||||
|
||||
if scrobbles_created:
|
||||
logger.info(
|
||||
f"Creating {len(scrobbles_created)} new scrobbles",
|
||||
extra={"scrobbles_created": scrobbles_created},
|
||||
)
|
||||
return scrobbles_created
|
||||
logger.info(f"No new scrobbles found in IMAP folders")
|
||||
return []
|
||||
96
vrobbler/apps/scrobbles/importers/tsv.py
Normal file
96
vrobbler/apps/scrobbles/importers/tsv.py
Normal file
@ -0,0 +1,96 @@
|
||||
import codecs
|
||||
import csv
|
||||
import logging
|
||||
|
||||
import pytz
|
||||
import requests
|
||||
from music.models import Track
|
||||
from scrobbles.constants import AsTsvColumn
|
||||
from scrobbles.models import Scrobble
|
||||
|
||||
from scrobbles.utils import timestamp_user_tz_to_utc
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
|
||||
"""Takes a path to a file of TSV data and imports it as past scrobbles"""
|
||||
new_scrobbles = []
|
||||
if not user_tz:
|
||||
user_tz = pytz.utc
|
||||
|
||||
is_os_file = "https://" not in file_path
|
||||
|
||||
if not is_os_file:
|
||||
r = requests.get(file_path)
|
||||
tsv_data = codecs.iterdecode(r.iter_lines(), "utf-8")
|
||||
else:
|
||||
tsv_data = open(file_path)
|
||||
|
||||
source = "Audioscrobbler File"
|
||||
rows = csv.reader(tsv_data, delimiter="\t")
|
||||
|
||||
rockbox_info = ""
|
||||
for row_num, row in enumerate(rows):
|
||||
if row_num in [0, 1, 2]:
|
||||
if "Rockbox" in row[0]:
|
||||
source = "Rockbox"
|
||||
rockbox_info += row[0] + "\n"
|
||||
continue
|
||||
if len(row) > 8:
|
||||
logger.info(
|
||||
"[skip] too many columns in row",
|
||||
extra={"row": row},
|
||||
)
|
||||
continue
|
||||
|
||||
track = Track.find_or_create(
|
||||
title=row[AsTsvColumn["TRACK_NAME"].value],
|
||||
musicbrainz_id=row[AsTsvColumn["MB_ID"].value],
|
||||
artist_name=row[AsTsvColumn["ARTIST_NAME"].value],
|
||||
album_name=row[AsTsvColumn["ALBUM_NAME"].value],
|
||||
)
|
||||
|
||||
# TODO Set all this up as constants
|
||||
if row[AsTsvColumn["COMPLETE"].value] == "S":
|
||||
logger.info(
|
||||
"[skip] track not finished",
|
||||
extra={
|
||||
"album_name": row[AsTsvColumn["ALBUM_NAME"].value],
|
||||
"artist_name": row[AsTsvColumn["ARTIST_NAME"].value],
|
||||
},
|
||||
)
|
||||
continue
|
||||
|
||||
timestamp = timestamp_user_tz_to_utc(
|
||||
int(row[AsTsvColumn["TIMESTAMP"].value]), user_tz
|
||||
)
|
||||
|
||||
new_scrobble = Scrobble(
|
||||
user_id=user_id,
|
||||
timestamp=timestamp,
|
||||
source=source,
|
||||
log={"rockbox_info": rockbox_info},
|
||||
track=track,
|
||||
played_to_completion=True,
|
||||
in_progress=False,
|
||||
media_type=Scrobble.MediaType.TRACK,
|
||||
)
|
||||
existing = Scrobble.objects.filter(
|
||||
timestamp=timestamp, track=track
|
||||
).first()
|
||||
if existing:
|
||||
logger.debug(f"Skipping existing scrobble {new_scrobble}")
|
||||
continue
|
||||
logger.debug(f"Queued scrobble {new_scrobble} for creation")
|
||||
new_scrobbles.append(new_scrobble)
|
||||
|
||||
if is_os_file:
|
||||
tsv_data.close()
|
||||
|
||||
created = Scrobble.objects.bulk_create(new_scrobbles)
|
||||
logger.info(
|
||||
f"Created {len(created)} scrobbles",
|
||||
extra={"created_scrobbles": created},
|
||||
)
|
||||
return created
|
||||
81
vrobbler/apps/scrobbles/importers/webdav.py
Normal file
81
vrobbler/apps/scrobbles/importers/webdav.py
Normal file
@ -0,0 +1,81 @@
|
||||
from books.koreader import fetch_file_from_webdav
|
||||
from profiles.models import UserProfile
|
||||
from scrobbles.models import KoReaderImport
|
||||
from scrobbles.tasks import process_koreader_import
|
||||
from webdav.client import get_webdav_client
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def import_from_webdav_for_all_users(restart=False):
|
||||
"""Grab a list of all users with WebDAV enabled and kickoff imports for them"""
|
||||
|
||||
# LastFmImport = apps.get_model("scrobbles", "LastFMImport")
|
||||
webdav_enabled_user_ids = UserProfile.objects.filter(
|
||||
webdav_url__isnull=False,
|
||||
webdav_user__isnull=False,
|
||||
webdav_pass__isnull=False,
|
||||
webdav_auto_import=True,
|
||||
).values_list("user_id", flat=True)
|
||||
logger.info(
|
||||
f"Start import of {webdav_enabled_user_ids.count()} webdav accounts"
|
||||
)
|
||||
|
||||
koreader_import_count = 0
|
||||
|
||||
for user_id in webdav_enabled_user_ids:
|
||||
webdav_client = get_webdav_client(user_id)
|
||||
|
||||
try:
|
||||
webdav_client.info("var/koreader/statistics.sqlite3")
|
||||
koreader_found = True
|
||||
except:
|
||||
koreader_found = False
|
||||
logger.info(
|
||||
"No koreader stats file found on webdav",
|
||||
extra={"user_id": user_id},
|
||||
)
|
||||
|
||||
if koreader_found:
|
||||
last_import = (
|
||||
KoReaderImport.objects.filter(
|
||||
user_id=user_id, processed_finished__isnull=False
|
||||
)
|
||||
.order_by("Processed_finished")
|
||||
.last()
|
||||
)
|
||||
|
||||
koreader_file_path = fetch_file_from_webdav(1)
|
||||
new_hash = get_file_md5_hash(koreader_file_path)
|
||||
old_hash = None
|
||||
if last_import:
|
||||
old_hash = last_import.file_md5_hash()
|
||||
|
||||
if old_hash and new_hash == old_hash:
|
||||
logger.info(
|
||||
"koreader stats file has not changed",
|
||||
extra={
|
||||
"user_id": user_id,
|
||||
"new_hash": new_hash,
|
||||
"old_hash": old_hash,
|
||||
"last_import_id": last_import.id,
|
||||
},
|
||||
)
|
||||
continue
|
||||
|
||||
koreader_import, created = KoReaderImport.objects.get_or_create(
|
||||
user_id=user_id, processed_finished__isnull=True
|
||||
)
|
||||
|
||||
if not created and not restart:
|
||||
logger.info(
|
||||
f"Not resuming failed KoReader import {koreader_import.id} for user {user_id}, use restart=True to restart"
|
||||
)
|
||||
continue
|
||||
|
||||
koreader_import.save_sqlite_file_to_self(koreader_file_path)
|
||||
|
||||
process_koreader_import.delay(koreader_import.id)
|
||||
koreader_import_count += 1
|
||||
return koreader_import_count
|
||||
Reference in New Issue
Block a user