secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[books] Try fixing the one-off import issue

Browse Source
This commit is contained in:
Colin Powell
2026-06-11 09:37:38 -04:00
parent de9b4ee9c1
commit e243fec679
3 changed files with 57 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
PROJECT.org
View File

@ -88,7 +88,7 @@ fetching and simple saving.
*** Metadata sources
**** Scraper
* Backlog [0/15] :vrobbler:project:personal:
* Backlog [1/17] :vrobbler:project:personal:
** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles:
:PROPERTIES:
:ID: 702462cf-d54b-48c6-8a7c-78b8de751deb
@ -544,6 +544,41 @@ log a warning and move on.
We should have a global view `/favorites/` that shows the logged in users's
favorited media objects.
** DONE [#B] Koreader imports only import single-page scrobbles the next day :bug:books:importers:
:PROPERTIES:
:ID: b50141fd-cda6-4a3a-afd3-cd8499e7523e
:END:
*** Description
When you read a single page in a book in Koreader and try to import it, the scrobble is only
created the day after, not on the day of the reading.
** STRT [#A] Fix bugs in celery tasks causing imports to fail :bug:celery:tasks:
:PROPERTIES:
:ID: d1171cb0-6413-44b8-a68a-019a4d2fb285
:END:
*** Description
Seems like all celery tasks are failing for different reasons except the chart
updates.
*** Errors
#+begin_src bash
KeyError: 'track'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 89, in _execute
return self.cursor.execute(sql, params)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
psycopg2.errors.UndefinedColumn: column music_track.artist_id does not exist
LINE 1: ..."."title", "music_track"."base_run_time_seconds", "music_tra...
^
HINT: Perhaps you meant to reference the column "music_track.artist_fk_id".
#+end_src
* Version 50.1 [1/1]
** DONE [#B] Fix bug in charts where only #1 is displayed :charts:templates:
:PROPERTIES:

40
vrobbler/apps/books/koreader.py
View File

@ -187,18 +187,20 @@ def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict:
book_ids_not_found.append(koreader_book_id)
continue
if "pages" not in book_map[koreader_book_id].keys():
book_map[koreader_book_id]["pages"] = {}
book_map[koreader_book_id].setdefault("pages", [])
page_number = page_row[KoReaderPageStatColumn.PAGE.value]
duration = page_row[KoReaderPageStatColumn.DURATION.value]
start_ts = page_row[KoReaderPageStatColumn.START_TIME.value]
book_map[koreader_book_id]["pages"][page_number] = {
"duration": duration,
"start_ts": start_ts,
"end_ts": start_ts + duration,
}
book_map[koreader_book_id]["pages"].append(
{
"page_number": page_number,
"duration": duration,
"start_ts": start_ts,
"end_ts": start_ts + duration,
}
)
if book_ids_not_found:
logger.info(f"Found pages for books not in file: {set(book_ids_not_found)}")
return book_map
@ -225,11 +227,12 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
pages_processed = 0
total_pages_read = len(book_map[koreader_book_id]["pages"])
ordered_pages = sorted(
book_map[koreader_book_id]["pages"].items(),
key=lambda x: x[1]["start_ts"],
book_map[koreader_book_id]["pages"],
key=lambda x: x["start_ts"],
)
for cur_page_number, stats in ordered_pages:
for stats in ordered_pages:
page_number = stats["page_number"]
pages_processed += 1
seconds_from_last_page = 0
@ -243,12 +246,14 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
)
end_of_reading = pages_processed == total_pages_read
big_jump_to_this_page = (cur_page_number - last_page_number) > 10
big_jump_to_this_page = (page_number - last_page_number) > 10
is_session_gap = seconds_from_last_page > SESSION_GAP_SECONDS
if (is_session_gap and not big_jump_to_this_page) or end_of_reading:
should_create_scrobble = True
if should_create_scrobble:
if not scrobble_page_data:
scrobble_page_data[page_number] = stats
scrobble_page_data = dict(
sorted(
scrobble_page_data.items(),
@ -276,13 +281,6 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
datetime.fromtimestamp(int(last_page.get("end_ts")))
)
# Adjust for Daylight Saving Time
# if timestamp.dst() == timedelta(
# 0
# ) or stop_timestamp.dst() == timedelta(0):
# timestamp = timestamp - timedelta(hours=1)
# stop_timestamp = stop_timestamp - timedelta(hours=1)
scrobble = Scrobble.objects.filter(
timestamp=timestamp,
book_id=book_id,
@ -291,7 +289,7 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
if not scrobble:
logger.info(
f"Queueing scrobble for {book_id}, page {cur_page_number}"
f"Queueing scrobble for {book_id}, page {page_number}"
)
log_data = {
"koreader_hash": book_dict.get("hash"),
@ -324,9 +322,9 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
scrobble_page_data = {}
# We accumulate pages for the scrobble until we should create a new one
scrobble_page_data[cur_page_number] = stats
scrobble_page_data[page_number] = stats
last_page_number = cur_page_number
last_page_number = page_number
prev_page_stats = stats
if pages_not_found:
logger.info(f"Pages not found for books: {set(pages_not_found)}")

4
vrobbler/apps/books/tests/conftest.py
View File

@ -32,8 +32,6 @@ class KoReaderBookRows:
DEFAULT_STR = "N/A"
DEFAULT_INT = 0
DEFAULT_TIME = 1703800469
BOOK_ROWS = []
PAGE_STATS_ROWS = []
def _gen_random_row(self, i):
wiggle = random.randrange(15)
@ -110,6 +108,8 @@ class KoReaderBookRows:
end_session = True
def __init__(self, book_count=0, **kwargs):
self.BOOK_ROWS = []
self.PAGE_STATS_ROWS = []
self._generate_random_book_rows(book_count)
self._generate_custom_book_row(**kwargs)
self._generate_random_page_stats_rows()