diff --git a/PROJECT.org b/PROJECT.org index 2b8860d..4d856a4 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/15] :vrobbler:project:personal: +* Backlog [1/17] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -544,6 +544,41 @@ log a warning and move on. We should have a global view `/favorites/` that shows the logged in users's favorited media objects. +** DONE [#B] Koreader imports only import single-page scrobbles the next day :bug:books:importers: +:PROPERTIES: +:ID: b50141fd-cda6-4a3a-afd3-cd8499e7523e +:END: + +*** Description + +When you read a single page in a book in Koreader and try to import it, the scrobble is only +created the day after, not on the day of the reading. + +** STRT [#A] Fix bugs in celery tasks causing imports to fail :bug:celery:tasks: +:PROPERTIES: +:ID: d1171cb0-6413-44b8-a68a-019a4d2fb285 +:END: + +*** Description + +Seems like all celery tasks are failing for different reasons except the chart +updates. + +*** Errors +#+begin_src bash +KeyError: 'track' + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): +File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 89, in _execute +return self.cursor.execute(sql, params) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +psycopg2.errors.UndefinedColumn: column music_track.artist_id does not exist +LINE 1: ..."."title", "music_track"."base_run_time_seconds", "music_tra... +^ +HINT: Perhaps you meant to reference the column "music_track.artist_fk_id". +#+end_src * Version 50.1 [1/1] ** DONE [#B] Fix bug in charts where only #1 is displayed :charts:templates: :PROPERTIES: diff --git a/vrobbler/apps/books/koreader.py b/vrobbler/apps/books/koreader.py index a9d0fad..209e6d9 100644 --- a/vrobbler/apps/books/koreader.py +++ b/vrobbler/apps/books/koreader.py @@ -187,18 +187,20 @@ def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict: book_ids_not_found.append(koreader_book_id) continue - if "pages" not in book_map[koreader_book_id].keys(): - book_map[koreader_book_id]["pages"] = {} + book_map[koreader_book_id].setdefault("pages", []) page_number = page_row[KoReaderPageStatColumn.PAGE.value] duration = page_row[KoReaderPageStatColumn.DURATION.value] start_ts = page_row[KoReaderPageStatColumn.START_TIME.value] - book_map[koreader_book_id]["pages"][page_number] = { - "duration": duration, - "start_ts": start_ts, - "end_ts": start_ts + duration, - } + book_map[koreader_book_id]["pages"].append( + { + "page_number": page_number, + "duration": duration, + "start_ts": start_ts, + "end_ts": start_ts + duration, + } + ) if book_ids_not_found: logger.info(f"Found pages for books not in file: {set(book_ids_not_found)}") return book_map @@ -225,11 +227,12 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl pages_processed = 0 total_pages_read = len(book_map[koreader_book_id]["pages"]) ordered_pages = sorted( - book_map[koreader_book_id]["pages"].items(), - key=lambda x: x[1]["start_ts"], + book_map[koreader_book_id]["pages"], + key=lambda x: x["start_ts"], ) - for cur_page_number, stats in ordered_pages: + for stats in ordered_pages: + page_number = stats["page_number"] pages_processed += 1 seconds_from_last_page = 0 @@ -243,12 +246,14 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl ) end_of_reading = pages_processed == total_pages_read - big_jump_to_this_page = (cur_page_number - last_page_number) > 10 + big_jump_to_this_page = (page_number - last_page_number) > 10 is_session_gap = seconds_from_last_page > SESSION_GAP_SECONDS if (is_session_gap and not big_jump_to_this_page) or end_of_reading: should_create_scrobble = True if should_create_scrobble: + if not scrobble_page_data: + scrobble_page_data[page_number] = stats scrobble_page_data = dict( sorted( scrobble_page_data.items(), @@ -276,13 +281,6 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl datetime.fromtimestamp(int(last_page.get("end_ts"))) ) - # Adjust for Daylight Saving Time - # if timestamp.dst() == timedelta( - # 0 - # ) or stop_timestamp.dst() == timedelta(0): - # timestamp = timestamp - timedelta(hours=1) - # stop_timestamp = stop_timestamp - timedelta(hours=1) - scrobble = Scrobble.objects.filter( timestamp=timestamp, book_id=book_id, @@ -291,7 +289,7 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl if not scrobble: logger.info( - f"Queueing scrobble for {book_id}, page {cur_page_number}" + f"Queueing scrobble for {book_id}, page {page_number}" ) log_data = { "koreader_hash": book_dict.get("hash"), @@ -324,9 +322,9 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl scrobble_page_data = {} # We accumulate pages for the scrobble until we should create a new one - scrobble_page_data[cur_page_number] = stats + scrobble_page_data[page_number] = stats - last_page_number = cur_page_number + last_page_number = page_number prev_page_stats = stats if pages_not_found: logger.info(f"Pages not found for books: {set(pages_not_found)}") diff --git a/vrobbler/apps/books/tests/conftest.py b/vrobbler/apps/books/tests/conftest.py index ca5a683..10ffda5 100644 --- a/vrobbler/apps/books/tests/conftest.py +++ b/vrobbler/apps/books/tests/conftest.py @@ -32,8 +32,6 @@ class KoReaderBookRows: DEFAULT_STR = "N/A" DEFAULT_INT = 0 DEFAULT_TIME = 1703800469 - BOOK_ROWS = [] - PAGE_STATS_ROWS = [] def _gen_random_row(self, i): wiggle = random.randrange(15) @@ -110,6 +108,8 @@ class KoReaderBookRows: end_session = True def __init__(self, book_count=0, **kwargs): + self.BOOK_ROWS = [] + self.PAGE_STATS_ROWS = [] self._generate_random_book_rows(book_count) self._generate_custom_book_row(**kwargs) self._generate_random_page_stats_rows()