diff --git a/vrobbler/apps/books/koreader.py b/vrobbler/apps/books/koreader.py index 209e6d9..7d92390 100644 --- a/vrobbler/apps/books/koreader.py +++ b/vrobbler/apps/books/koreader.py @@ -1,9 +1,8 @@ import logging import re import sqlite3 -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone from enum import Enum -from zoneinfo import ZoneInfo import requests from books.constants import BOOKS_TITLES_TO_IGNORE @@ -174,7 +173,7 @@ def build_book_map(rows) -> dict: return book_id_map -def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict: +def build_page_data(page_rows: list, book_map: dict) -> dict: """Given rows of page data from KoReader, parse each row and build scrobbles for our user, loading the page data into the page_data field on the scrobble instance. @@ -275,10 +274,14 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl continue timestamp = user.profile.get_timestamp_with_tz( - datetime.fromtimestamp(int(first_page.get("start_ts"))) + datetime.fromtimestamp( + int(first_page.get("start_ts")) + ) ) stop_timestamp = user.profile.get_timestamp_with_tz( - datetime.fromtimestamp(int(last_page.get("end_ts"))) + datetime.fromtimestamp( + int(last_page.get("end_ts")) + ) ) scrobble = Scrobble.objects.filter( @@ -326,6 +329,80 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl last_page_number = page_number prev_page_stats = stats + + # Handle leftover pages that never triggered a session gap + if scrobble_page_data: + scrobble_page_data = dict( + sorted( + scrobble_page_data.items(), + key=lambda x: x[1]["start_ts"], + ) + ) + try: + first_page = scrobble_page_data.get( + list(scrobble_page_data.keys())[0] + ) + last_page = scrobble_page_data.get( + list(scrobble_page_data.keys())[-1] + ) + except IndexError: + logger.error( + "Could not process book, no page data found", + extra={"scrobble_page_data": scrobble_page_data}, + ) + continue + + playback_position_seconds = sum( + p["duration"] for p in scrobble_page_data.values() + ) + + timestamp = user.profile.get_timestamp_with_tz( + datetime.fromtimestamp( + int(first_page.get("start_ts")) + ) + ) + stop_timestamp = user.profile.get_timestamp_with_tz( + datetime.fromtimestamp( + int(last_page.get("end_ts")) + ) + ) + + scrobble = Scrobble.objects.filter( + timestamp=timestamp, + book_id=book_id, + user_id=user.id, + ).first() + + if not scrobble: + logger.info( + f"Queueing scrobble for {book_id}, page {list(scrobble_page_data.keys())[0]}" + ) + log_data = { + "koreader_hash": book_dict.get("hash"), + "page_data": scrobble_page_data, + "pages_read": len(scrobble_page_data.keys()), + } + if hasattr(timestamp.tzinfo, "tzname"): + tz = timestamp.tzinfo.tzname + if hasattr(timestamp.tzinfo, "name"): + tz = timestamp.tzinfo.name + scrobbles_to_create.append( + Scrobble( + book_id=book_id, + user_id=user.id, + source="KOReader", + media_type=Scrobble.MediaType.BOOK, + timestamp=timestamp, + log=log_data, + stop_timestamp=stop_timestamp, + playback_position_seconds=playback_position_seconds, + in_progress=False, + played_to_completion=True, + long_play_complete=False, + timezone=tz, + ) + ) + if pages_not_found: logger.info(f"Pages not found for books: {set(pages_not_found)}") return scrobbles_to_create @@ -355,9 +432,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: new_scrobbles = [] user = User.objects.filter(id=user_id).first() - tz = ZoneInfo("UTC") - if user: - tz = user.profile.tzinfo is_os_file = "https://" not in file_path if is_os_file: @@ -373,7 +447,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: book_map = build_page_data( cur.execute("SELECT * from page_stat_data ORDER BY id_book, start_time"), book_map, - tz, ) new_scrobbles = build_scrobbles_from_book_map(book_map, user) else: @@ -390,7 +463,7 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: _sqlite_bytes(file_path), max_buffer_size=1_048_576 ): if table_name == "page_stat_data": - book_map = build_page_data(rows, book_map, tz) + book_map = build_page_data(rows, book_map) new_scrobbles = build_scrobbles_from_book_map(book_map, user) logger.info(f"Creating {len(new_scrobbles)} new scrobbles") diff --git a/vrobbler/apps/books/models.py b/vrobbler/apps/books/models.py index 1547541..a904613 100644 --- a/vrobbler/apps/books/models.py +++ b/vrobbler/apps/books/models.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from datetime import datetime from typing import Optional from uuid import uuid4 +from zoneinfo import ZoneInfo import requests from books.constants import MediaSourceTag, READCOMICSONLINE_URL @@ -472,8 +473,11 @@ class Book(LongPlayScrobblableMixin): if scrobble.logdata.page_data: for page, data in scrobble.logdata.page_data.items(): if convert_timestamps: - data["start_ts"] = datetime.fromtimestamp(data["start_ts"]) - data["end_ts"] = datetime.fromtimestamp(data["end_ts"]) + tz = None + if scrobble.timezone: + tz = ZoneInfo(scrobble.timezone) + data["start_ts"] = datetime.fromtimestamp(data["start_ts"], tz=tz) + data["end_ts"] = datetime.fromtimestamp(data["end_ts"], tz=tz) pages[page] = data sorted_pages = OrderedDict( sorted(pages.items(), key=lambda x: x[1]["start_ts"]) diff --git a/vrobbler/apps/books/tests/test_koreader.py b/vrobbler/apps/books/tests/test_koreader.py index b45baa8..b9435ca 100644 --- a/vrobbler/apps/books/tests/test_koreader.py +++ b/vrobbler/apps/books/tests/test_koreader.py @@ -44,8 +44,12 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re book_map = build_page_data(koreader_rows.PAGE_STATS_ROWS, book_map) scrobbles = build_scrobbles_from_book_map(book_map, demo_user) - # Corresponds to number of sessions per book ( 20 pages per session, 120 +/- 15 pages read ) - expected_scrobbles = 6 * len(book_map.keys()) + # The test data generator adds the session-gap 3600s AFTER the trigger page + # (not before), so the first session includes 21 pages (1-21), and each + # subsequent session has 20 until the last. The last trigger page (120) was + # previously orphaned by the loop structure; the post-loop fix now creates a + # scrobble for it. + expected_scrobbles = 7 * len(book_map.keys()) assert len(scrobbles) == expected_scrobbles assert len(scrobbles[0].logdata.page_data.keys()) == 21 assert len(scrobbles[1].logdata.page_data.keys()) == 20 @@ -53,6 +57,7 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re assert len(scrobbles[3].logdata.page_data.keys()) == 20 assert len(scrobbles[4].logdata.page_data.keys()) == 20 assert len(scrobbles[5].logdata.page_data.keys()) == 18 + assert len(scrobbles[6].logdata.page_data.keys()) == 1 def test_get_author_str_from_row(): diff --git a/vrobbler/apps/profiles/models.py b/vrobbler/apps/profiles/models.py index 25ddb9f..cdfa413 100644 --- a/vrobbler/apps/profiles/models.py +++ b/vrobbler/apps/profiles/models.py @@ -140,6 +140,11 @@ class UserProfile(TimeStampedModel): return history def get_timestamp_with_tz(self, timestamp): + from django.conf import settings + + server_tz = ZoneInfo(settings.TIME_ZONE) + ref_dt = timestamp if timestamp.tzinfo is not None else timestamp.replace(tzinfo=server_tz) + timezone = self.tzinfo if self.timezone_change_log: change_list = self.historic_timezone_changes @@ -150,13 +155,13 @@ class UserProfile(TimeStampedModel): end = None if end: - if start <= timestamp.replace(tzinfo=end.timezone) <= end: + if start <= ref_dt <= end: timezone = start.timezone else: - if start <= timestamp.replace(tzinfo=start.timezone): + if start <= ref_dt: timezone = start.timezone - return timestamp.replace(tzinfo=timezone) + return ref_dt.astimezone(timezone) def adjust_timezone_of_scrobbles(self, commit=False): current_dt = None