secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[books] Fix koreader imports, maybe forever

Browse Source
This commit is contained in:
Colin Powell
2026-06-11 18:38:51 -04:00
parent a171192a6f
commit 410da163fe
4 changed files with 104 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
vrobbler/apps/books/koreader.py
View File

@ -1,9 +1,8 @@
import logging
import re
import sqlite3
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from enum import Enum
from zoneinfo import ZoneInfo
import requests
from books.constants import BOOKS_TITLES_TO_IGNORE
@ -174,7 +173,7 @@ def build_book_map(rows) -> dict:
return book_id_map
def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict:
def build_page_data(page_rows: list, book_map: dict) -> dict:
"""Given rows of page data from KoReader, parse each row and build
scrobbles for our user, loading the page data into the page_data
field on the scrobble instance.
@ -275,10 +274,14 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
continue
timestamp = user.profile.get_timestamp_with_tz(
datetime.fromtimestamp(int(first_page.get("start_ts")))
datetime.fromtimestamp(
int(first_page.get("start_ts"))
)
)
stop_timestamp = user.profile.get_timestamp_with_tz(
datetime.fromtimestamp(int(last_page.get("end_ts")))
datetime.fromtimestamp(
int(last_page.get("end_ts"))
)
)
scrobble = Scrobble.objects.filter(
@ -326,6 +329,80 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
last_page_number = page_number
prev_page_stats = stats
# Handle leftover pages that never triggered a session gap
if scrobble_page_data:
scrobble_page_data = dict(
sorted(
scrobble_page_data.items(),
key=lambda x: x[1]["start_ts"],
)
)
try:
first_page = scrobble_page_data.get(
list(scrobble_page_data.keys())[0]
)
last_page = scrobble_page_data.get(
list(scrobble_page_data.keys())[-1]
)
except IndexError:
logger.error(
"Could not process book, no page data found",
extra={"scrobble_page_data": scrobble_page_data},
)
continue
playback_position_seconds = sum(
p["duration"] for p in scrobble_page_data.values()
)
timestamp = user.profile.get_timestamp_with_tz(
datetime.fromtimestamp(
int(first_page.get("start_ts"))
)
)
stop_timestamp = user.profile.get_timestamp_with_tz(
datetime.fromtimestamp(
int(last_page.get("end_ts"))
)
)
scrobble = Scrobble.objects.filter(
timestamp=timestamp,
book_id=book_id,
user_id=user.id,
).first()
if not scrobble:
logger.info(
f"Queueing scrobble for {book_id}, page {list(scrobble_page_data.keys())[0]}"
)
log_data = {
"koreader_hash": book_dict.get("hash"),
"page_data": scrobble_page_data,
"pages_read": len(scrobble_page_data.keys()),
}
if hasattr(timestamp.tzinfo, "tzname"):
tz = timestamp.tzinfo.tzname
if hasattr(timestamp.tzinfo, "name"):
tz = timestamp.tzinfo.name
scrobbles_to_create.append(
Scrobble(
book_id=book_id,
user_id=user.id,
source="KOReader",
media_type=Scrobble.MediaType.BOOK,
timestamp=timestamp,
log=log_data,
stop_timestamp=stop_timestamp,
playback_position_seconds=playback_position_seconds,
in_progress=False,
played_to_completion=True,
long_play_complete=False,
timezone=tz,
)
)
if pages_not_found:
logger.info(f"Pages not found for books: {set(pages_not_found)}")
return scrobbles_to_create
@ -355,9 +432,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
new_scrobbles = []
user = User.objects.filter(id=user_id).first()
tz = ZoneInfo("UTC")
if user:
tz = user.profile.tzinfo
is_os_file = "https://" not in file_path
if is_os_file:
@ -373,7 +447,6 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
book_map = build_page_data(
cur.execute("SELECT * from page_stat_data ORDER BY id_book, start_time"),
book_map,
tz,
)
new_scrobbles = build_scrobbles_from_book_map(book_map, user)
else:
@ -390,7 +463,7 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
_sqlite_bytes(file_path), max_buffer_size=1_048_576
):
if table_name == "page_stat_data":
book_map = build_page_data(rows, book_map, tz)
book_map = build_page_data(rows, book_map)
new_scrobbles = build_scrobbles_from_book_map(book_map, user)
logger.info(f"Creating {len(new_scrobbles)} new scrobbles")

8
vrobbler/apps/books/models.py
View File

@ -4,6 +4,7 @@ from dataclasses import dataclass
from datetime import datetime
from typing import Optional
from uuid import uuid4
from zoneinfo import ZoneInfo
import requests
from books.constants import MediaSourceTag, READCOMICSONLINE_URL
@ -472,8 +473,11 @@ class Book(LongPlayScrobblableMixin):
if scrobble.logdata.page_data:
for page, data in scrobble.logdata.page_data.items():
if convert_timestamps:
data["start_ts"] = datetime.fromtimestamp(data["start_ts"])
data["end_ts"] = datetime.fromtimestamp(data["end_ts"])
tz = None
if scrobble.timezone:
tz = ZoneInfo(scrobble.timezone)
data["start_ts"] = datetime.fromtimestamp(data["start_ts"], tz=tz)
data["end_ts"] = datetime.fromtimestamp(data["end_ts"], tz=tz)
pages[page] = data
sorted_pages = OrderedDict(
sorted(pages.items(), key=lambda x: x[1]["start_ts"])

9
vrobbler/apps/books/tests/test_koreader.py
View File

@ -44,8 +44,12 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re
book_map = build_page_data(koreader_rows.PAGE_STATS_ROWS, book_map)
scrobbles = build_scrobbles_from_book_map(book_map, demo_user)
# Corresponds to number of sessions per book ( 20 pages per session, 120 +/- 15 pages read )
expected_scrobbles = 6 * len(book_map.keys())
# The test data generator adds the session-gap 3600s AFTER the trigger page
# (not before), so the first session includes 21 pages (1-21), and each
# subsequent session has 20 until the last. The last trigger page (120) was
# previously orphaned by the loop structure; the post-loop fix now creates a
# scrobble for it.
expected_scrobbles = 7 * len(book_map.keys())
assert len(scrobbles) == expected_scrobbles
assert len(scrobbles[0].logdata.page_data.keys()) == 21
assert len(scrobbles[1].logdata.page_data.keys()) == 20
@ -53,6 +57,7 @@ def test_build_scrobbles_from_pages(get_mock, koreader_rows, demo_user, valid_re
assert len(scrobbles[3].logdata.page_data.keys()) == 20
assert len(scrobbles[4].logdata.page_data.keys()) == 20
assert len(scrobbles[5].logdata.page_data.keys()) == 18
assert len(scrobbles[6].logdata.page_data.keys()) == 1
def test_get_author_str_from_row():

11
vrobbler/apps/profiles/models.py
View File

@ -140,6 +140,11 @@ class UserProfile(TimeStampedModel):
return history
def get_timestamp_with_tz(self, timestamp):
from django.conf import settings
server_tz = ZoneInfo(settings.TIME_ZONE)
ref_dt = timestamp if timestamp.tzinfo is not None else timestamp.replace(tzinfo=server_tz)
timezone = self.tzinfo
if self.timezone_change_log:
change_list = self.historic_timezone_changes
@ -150,13 +155,13 @@ class UserProfile(TimeStampedModel):
end = None
if end:
if start <= timestamp.replace(tzinfo=end.timezone) <= end:
if start <= ref_dt <= end:
timezone = start.timezone
else:
if start <= timestamp.replace(tzinfo=start.timezone):
if start <= ref_dt:
timezone = start.timezone
return timestamp.replace(tzinfo=timezone)
return ref_dt.astimezone(timezone)
def adjust_timezone_of_scrobbles(self, commit=False):
current_dt = None