[books] Fix koreader imports, maybe forever

vrobbler/apps/books/koreader.py

@@ -1,9 +1,8 @@
 import logging
 import re
 import sqlite3
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from enum import Enum
-from zoneinfo import ZoneInfo
 
 import requests
 from books.constants import BOOKS_TITLES_TO_IGNORE
@@ -174,7 +173,7 @@ def build_book_map(rows) -> dict:
     return book_id_map
 
 
-def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict:
+def build_page_data(page_rows: list, book_map: dict) -> dict:
     """Given rows of page data from KoReader, parse each row and build
     scrobbles for our user, loading the page data into the page_data
     field on the scrobble instance.
@@ -275,10 +274,14 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
                     continue
 
                 timestamp = user.profile.get_timestamp_with_tz(
-                    datetime.fromtimestamp(int(first_page.get("start_ts")))
+                    datetime.fromtimestamp(
+                        int(first_page.get("start_ts"))
+                    )
                 )
                 stop_timestamp = user.profile.get_timestamp_with_tz(
-                    datetime.fromtimestamp(int(last_page.get("end_ts")))
+                    datetime.fromtimestamp(
+                        int(last_page.get("end_ts"))
+                    )
                 )
 
                 scrobble = Scrobble.objects.filter(
@@ -326,6 +329,80 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
 
             last_page_number = page_number
             prev_page_stats = stats
+
+        # Handle leftover pages that never triggered a session gap
+        if scrobble_page_data:
+            scrobble_page_data = dict(
+                sorted(
+                    scrobble_page_data.items(),
+                    key=lambda x: x[1]["start_ts"],
+                )
+            )
+            try:
+                first_page = scrobble_page_data.get(
+                    list(scrobble_page_data.keys())[0]
+                )
+                last_page = scrobble_page_data.get(
+                    list(scrobble_page_data.keys())[-1]
+                )
+            except IndexError:
+                logger.error(
+                    "Could not process book, no page data found",
+                    extra={"scrobble_page_data": scrobble_page_data},
+                )
+                continue
+
+            playback_position_seconds = sum(
+                p["duration"] for p in scrobble_page_data.values()
+            )
+
+            timestamp = user.profile.get_timestamp_with_tz(
+                datetime.fromtimestamp(
+                    int(first_page.get("start_ts"))
+                )
+            )
+            stop_timestamp = user.profile.get_timestamp_with_tz(
+                datetime.fromtimestamp(
+                    int(last_page.get("end_ts"))
+                )
+            )
+
+            scrobble = Scrobble.objects.filter(
+                timestamp=timestamp,
+                book_id=book_id,
+                user_id=user.id,
+            ).first()
+
+            if not scrobble:
+                logger.info(
+                    f"Queueing scrobble for {book_id}, page {list(scrobble_page_data.keys())[0]}"
+                )
+                log_data = {
+                    "koreader_hash": book_dict.get("hash"),
+                    "page_data": scrobble_page_data,
+                    "pages_read": len(scrobble_page_data.keys()),
+                }
+                if hasattr(timestamp.tzinfo, "tzname"):
+                    tz = timestamp.tzinfo.tzname
+                if hasattr(timestamp.tzinfo, "name"):
+                    tz = timestamp.tzinfo.name
+                scrobbles_to_create.append(
+                    Scrobble(
+                        book_id=book_id,
+                        user_id=user.id,
+                        source="KOReader",
+                        media_type=Scrobble.MediaType.BOOK,
+                        timestamp=timestamp,
+                        log=log_data,
+                        stop_timestamp=stop_timestamp,
+                        playback_position_seconds=playback_position_seconds,
+                        in_progress=False,
+                        played_to_completion=True,
+                        long_play_complete=False,
+                        timezone=tz,
+                    )
+                )
+
     if pages_not_found:
         logger.info(f"Pages not found for books: {set(pages_not_found)}")
     return scrobbles_to_create
@@ -355,9 +432,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
 
     new_scrobbles = []
     user = User.objects.filter(id=user_id).first()
-    tz = ZoneInfo("UTC")
-    if user:
-        tz = user.profile.tzinfo
 
     is_os_file = "https://" not in file_path
     if is_os_file:
@@ -373,7 +447,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
         book_map = build_page_data(
             cur.execute("SELECT * from page_stat_data ORDER BY id_book, start_time"),
             book_map,
-            tz,
         )
         new_scrobbles = build_scrobbles_from_book_map(book_map, user)
     else:
@@ -390,7 +463,7 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
             _sqlite_bytes(file_path), max_buffer_size=1_048_576
         ):
             if table_name == "page_stat_data":
-                book_map = build_page_data(rows, book_map, tz)
+                book_map = build_page_data(rows, book_map)
                 new_scrobbles = build_scrobbles_from_book_map(book_map, user)
 
     logger.info(f"Creating {len(new_scrobbles)} new scrobbles")

vrobbler/apps/books/models.py

@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from datetime import datetime
 from typing import Optional
 from uuid import uuid4
+from zoneinfo import ZoneInfo
 
 import requests
 from books.constants import MediaSourceTag, READCOMICSONLINE_URL
@@ -472,8 +473,11 @@ class Book(LongPlayScrobblableMixin):
             if scrobble.logdata.page_data:
                 for page, data in scrobble.logdata.page_data.items():
                     if convert_timestamps:
-                        data["start_ts"] = datetime.fromtimestamp(data["start_ts"])
+                        tz = None
-                        data["end_ts"] = datetime.fromtimestamp(data["end_ts"])
+                        if scrobble.timezone:
+                            tz = ZoneInfo(scrobble.timezone)
+                        data["start_ts"] = datetime.fromtimestamp(data["start_ts"], tz=tz)
+                        data["end_ts"] = datetime.fromtimestamp(data["end_ts"], tz=tz)
                     pages[page] = data
         sorted_pages = OrderedDict(
             sorted(pages.items(), key=lambda x: x[1]["start_ts"])

vrobbler/apps/books/tests/test_koreader.py

@@ -44,8 +44,12 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re
     book_map = build_page_data(koreader_rows.PAGE_STATS_ROWS, book_map)
 
     scrobbles = build_scrobbles_from_book_map(book_map, demo_user)
-    # Corresponds to number of sessions per book ( 20 pages per session, 120 +/- 15 pages read )
+    # The test data generator adds the session-gap 3600s AFTER the trigger page
-    expected_scrobbles = 6 * len(book_map.keys())
+    # (not before), so the first session includes 21 pages (1-21), and each
+    # subsequent session has 20 until the last. The last trigger page (120) was
+    # previously orphaned by the loop structure; the post-loop fix now creates a
+    # scrobble for it.
+    expected_scrobbles = 7 * len(book_map.keys())
     assert len(scrobbles) == expected_scrobbles
     assert len(scrobbles[0].logdata.page_data.keys()) == 21
     assert len(scrobbles[1].logdata.page_data.keys()) == 20
@@ -53,6 +57,7 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re
     assert len(scrobbles[3].logdata.page_data.keys()) == 20
     assert len(scrobbles[4].logdata.page_data.keys()) == 20
     assert len(scrobbles[5].logdata.page_data.keys()) == 18
+    assert len(scrobbles[6].logdata.page_data.keys()) == 1
 
 
 def test_get_author_str_from_row():

vrobbler/apps/profiles/models.py

@@ -140,6 +140,11 @@ class UserProfile(TimeStampedModel):
         return history
 
     def get_timestamp_with_tz(self, timestamp):
+        from django.conf import settings
+
+        server_tz = ZoneInfo(settings.TIME_ZONE)
+        ref_dt = timestamp if timestamp.tzinfo is not None else timestamp.replace(tzinfo=server_tz)
+
         timezone = self.tzinfo
         if self.timezone_change_log:
             change_list = self.historic_timezone_changes
@@ -150,13 +155,13 @@ class UserProfile(TimeStampedModel):
                     end = None
 
                 if end:
-                    if start <= timestamp.replace(tzinfo=end.timezone) <= end:
+                    if start <= ref_dt <= end:
                         timezone = start.timezone
                 else:
-                    if start <= timestamp.replace(tzinfo=start.timezone):
+                    if start <= ref_dt:
                         timezone = start.timezone
 
-        return timestamp.replace(tzinfo=timezone)
+        return ref_dt.astimezone(timezone)
 
     def adjust_timezone_of_scrobbles(self, commit=False):
         current_dt = None