[books] Try fixing the one-off import issue

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [0/15] :vrobbler:project:personal:
+* Backlog [1/17] :vrobbler:project:personal:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles:
 :PROPERTIES:
 :ID:       702462cf-d54b-48c6-8a7c-78b8de751deb
@@ -544,6 +544,41 @@ log a warning and move on.
 We should have a global view `/favorites/` that shows the logged in users's
 favorited media objects.
 
+** DONE [#B] Koreader imports only import single-page scrobbles the next day :bug:books:importers:
+:PROPERTIES:
+:ID:       b50141fd-cda6-4a3a-afd3-cd8499e7523e
+:END:
+
+*** Description
+
+When you read a single page in a book in Koreader and try to import it, the scrobble is only
+created the day after, not on the day of the reading.
+
+** STRT [#A] Fix bugs in celery tasks causing imports to fail :bug:celery:tasks:
+:PROPERTIES:
+:ID:       d1171cb0-6413-44b8-a68a-019a4d2fb285
+:END:
+
+*** Description
+
+Seems like all celery tasks are failing for different reasons except the chart
+updates.
+
+*** Errors
+#+begin_src bash
+KeyError: 'track'
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+File "/usr/local/lib/python3.11/site-packages/django/db/backends/utils.py", line 89, in _execute
+return self.cursor.execute(sql, params)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+psycopg2.errors.UndefinedColumn: column music_track.artist_id does not exist
+LINE 1: ..."."title", "music_track"."base_run_time_seconds", "music_tra...
+^
+HINT: Perhaps you meant to reference the column "music_track.artist_fk_id".
+#+end_src
 * Version 50.1 [1/1]
 ** DONE [#B] Fix bug in charts where only #1 is displayed :charts:templates:
 :PROPERTIES:

vrobbler/apps/books/koreader.py

@@ -187,18 +187,20 @@ def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict:
             book_ids_not_found.append(koreader_book_id)
             continue
 
-        if "pages" not in book_map[koreader_book_id].keys():
+        book_map[koreader_book_id].setdefault("pages", [])
-            book_map[koreader_book_id]["pages"] = {}
 
         page_number = page_row[KoReaderPageStatColumn.PAGE.value]
         duration = page_row[KoReaderPageStatColumn.DURATION.value]
         start_ts = page_row[KoReaderPageStatColumn.START_TIME.value]
 
-        book_map[koreader_book_id]["pages"][page_number] = {
+        book_map[koreader_book_id]["pages"].append(
-            "duration": duration,
+            {
-            "start_ts": start_ts,
+                "page_number": page_number,
-            "end_ts": start_ts + duration,
+                "duration": duration,
-        }
+                "start_ts": start_ts,
+                "end_ts": start_ts + duration,
+            }
+        )
     if book_ids_not_found:
         logger.info(f"Found pages for books not in file: {set(book_ids_not_found)}")
     return book_map
@@ -225,11 +227,12 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
         pages_processed = 0
         total_pages_read = len(book_map[koreader_book_id]["pages"])
         ordered_pages = sorted(
-            book_map[koreader_book_id]["pages"].items(),
+            book_map[koreader_book_id]["pages"],
-            key=lambda x: x[1]["start_ts"],
+            key=lambda x: x["start_ts"],
         )
 
-        for cur_page_number, stats in ordered_pages:
+        for stats in ordered_pages:
+            page_number = stats["page_number"]
             pages_processed += 1
 
             seconds_from_last_page = 0
@@ -243,12 +246,14 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
             )
 
             end_of_reading = pages_processed == total_pages_read
-            big_jump_to_this_page = (cur_page_number - last_page_number) > 10
+            big_jump_to_this_page = (page_number - last_page_number) > 10
             is_session_gap = seconds_from_last_page > SESSION_GAP_SECONDS
             if (is_session_gap and not big_jump_to_this_page) or end_of_reading:
                 should_create_scrobble = True
 
             if should_create_scrobble:
+                if not scrobble_page_data:
+                    scrobble_page_data[page_number] = stats
                 scrobble_page_data = dict(
                     sorted(
                         scrobble_page_data.items(),
@@ -276,13 +281,6 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
                     datetime.fromtimestamp(int(last_page.get("end_ts")))
                 )
 
-                # Adjust for Daylight Saving Time
-                # if timestamp.dst() == timedelta(
-                #    0
-                # ) or stop_timestamp.dst() == timedelta(0):
-                #    timestamp = timestamp - timedelta(hours=1)
-                #    stop_timestamp = stop_timestamp - timedelta(hours=1)
-
                 scrobble = Scrobble.objects.filter(
                     timestamp=timestamp,
                     book_id=book_id,
@@ -291,7 +289,7 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
 
                 if not scrobble:
                     logger.info(
-                        f"Queueing scrobble for {book_id}, page {cur_page_number}"
+                        f"Queueing scrobble for {book_id}, page {page_number}"
                     )
                     log_data = {
                         "koreader_hash": book_dict.get("hash"),
@@ -324,9 +322,9 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl
                 scrobble_page_data = {}
 
             # We accumulate pages for the scrobble until we should create a new one
-            scrobble_page_data[cur_page_number] = stats
+            scrobble_page_data[page_number] = stats
 
-            last_page_number = cur_page_number
+            last_page_number = page_number
             prev_page_stats = stats
     if pages_not_found:
         logger.info(f"Pages not found for books: {set(pages_not_found)}")

vrobbler/apps/books/tests/conftest.py

@@ -32,8 +32,6 @@ class KoReaderBookRows:
     DEFAULT_STR = "N/A"
     DEFAULT_INT = 0
     DEFAULT_TIME = 1703800469
-    BOOK_ROWS = []
-    PAGE_STATS_ROWS = []
 
     def _gen_random_row(self, i):
         wiggle = random.randrange(15)
@@ -110,6 +108,8 @@ class KoReaderBookRows:
                     end_session = True
 
     def __init__(self, book_count=0, **kwargs):
+        self.BOOK_ROWS = []
+        self.PAGE_STATS_ROWS = []
         self._generate_random_book_rows(book_count)
         self._generate_custom_book_row(**kwargs)
         self._generate_random_page_stats_rows()