diff --git a/vrobbler/apps/books/koreader.py b/vrobbler/apps/books/koreader.py index 4d6338b..725e8f5 100644 --- a/vrobbler/apps/books/koreader.py +++ b/vrobbler/apps/books/koreader.py @@ -62,6 +62,7 @@ def get_author_str_from_row(row): # Strip middle initials, OpenLibrary often fails with these return re.sub(" [A-Z]. ", " ", ko_authors) + def lookup_or_create_authors_from_author_str(ko_author_str: str) -> list: """Takes a string of authors from KoReader and returns a list of Authors from our database @@ -72,9 +73,7 @@ def lookup_or_create_authors_from_author_str(ko_author_str: str) -> list: logger.debug(f"Looking up author {author_str}") # KoReader gave us nothing, bail if author_str == "N/A": - logger.warn( - f"KoReader author string is N/A, no authors to find" - ) + logger.warn(f"KoReader author string is N/A, no authors to find") continue author = Author.objects.filter(name=author_str).first() @@ -106,15 +105,14 @@ def create_book_from_row(row: list): book.fix_metadata() # Add authors - author_list = lookup_or_create_authors_from_author_str( - author_str - ) + author_list = lookup_or_create_authors_from_author_str(author_str) if author_list: book.authors.add(*author_list) # self._lookup_authors return book + def build_book_map(rows) -> dict: """Given an interable of sqlite rows from the books table, lookup existing books, create ones that don't exist, and return a mapping of koreader IDs to @@ -134,9 +132,7 @@ def build_book_map(rows) -> dict: book.refresh_from_db() total_seconds = 0 if book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]: - total_seconds = book_row[ - KoReaderBookColumn.TOTAL_READ_TIME.value - ] + total_seconds = book_row[KoReaderBookColumn.TOTAL_READ_TIME.value] book_id_map[book_row[KoReaderBookColumn.ID.value]] = { "book_id": book.id, @@ -144,13 +140,19 @@ def build_book_map(rows) -> dict: } return book_id_map -def build_page_data(page_rows: list, book_map: dict, user_tz = None) -> dict: + +def build_page_data(page_rows: list, book_map: dict, user_tz=None) -> dict: """Given rows of page data from KoReader, parse each row and build scrobbles for our user, loading the page data into the page_data field on the scrobble instance. """ for page_row in page_rows: koreader_book_id = page_row[KoReaderPageStatColumn.ID_BOOK.value] + if koreader_book_id not in book_map.keys(): + logger.info( + f"Found pages for book ID {koreader_book_id} not in our history, skipping" + ) + continue if "pages" not in book_map[koreader_book_id].keys(): book_map[koreader_book_id]["pages"] = {} @@ -164,13 +166,20 @@ def build_page_data(page_rows: list, book_map: dict, user_tz = None) -> dict: page_number = page_row[KoReaderPageStatColumn.PAGE.value] duration = page_row[KoReaderPageStatColumn.DURATION.value] start_ts = page_row[KoReaderPageStatColumn.START_TIME.value] - if user_tz: - start_ts = timestamp_user_tz_to_utc( - page_row[KoReaderPageStatColumn.START_TIME.value], - pytz.timezone(user_tz), - ).timestamp() - else: - logger.warning(f"Page data built with out user timezone, defaulting to UTC") + # TODO Not sure if this is doing nothing, as timestamps are already assumed in user TZ + # Maybe we want to save datetime strings with TZ info here? + # if user_tz: + # start_ts = ( + # datetime.utcfromtimestamp( + # int(page_row[KoReaderPageStatColumn.START_TIME.value]) + # ) + # .replace(tzinfo=user_tz) + # .timestamp() + # ) + # else: + # logger.warning( + # f"Page data built with out user timezone, defaulting to UTC" + # ) book_map[koreader_book_id]["pages"][page_number] = { "duration": duration, @@ -179,7 +188,10 @@ def build_page_data(page_rows: list, book_map: dict, user_tz = None) -> dict: } return book_map -def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobble"]: + +def build_scrobbles_from_book_map( + book_map: dict, user: "User" +) -> list["Scrobble"]: Scrobble = apps.get_model("scrobbles", "Scrobble") scrobbles_to_create = [] @@ -198,9 +210,7 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl pages_processed = 0 total_pages = len(book_map[koreader_book_id]["pages"]) - for page_number, stats in book_map[koreader_book_id][ - "pages" - ].items(): + for page_number, stats in book_map[koreader_book_id]["pages"].items(): pages_processed += 1 # Accumulate our page data for this scrobble scrobble_page_data[page_number] = stats @@ -210,24 +220,39 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl seconds_from_last_page = stats.get( "end_ts" ) - prev_page_stats.get("start_ts") - playback_position_seconds = ( - playback_position_seconds + stats.get("duration") + playback_position_seconds = playback_position_seconds + stats.get( + "duration" ) end_of_reading = pages_processed == total_pages - if ( - seconds_from_last_page > SESSION_GAP_SECONDS - or end_of_reading - ): + if seconds_from_last_page > SESSION_GAP_SECONDS or end_of_reading: should_create_scrobble = True - logger.info(f"Book {koreader_book_id} - {page_number} {seconds_from_last_page} read seconds") + logger.info( + f"Book {koreader_book_id} - {page_number} {seconds_from_last_page} read seconds" + ) if should_create_scrobble: first_page_in_scrobble = list(scrobble_page_data.keys())[0] - timestamp = timestamp_user_tz_to_utc( - int(scrobble_page_data.get(first_page_in_scrobble).get("start_ts")), - pytz.timezone(user.profile.timezone), + start_ts = int( + scrobble_page_data.get(first_page_in_scrobble).get( + "start_ts" + ) ) + timestamp = datetime.fromtimestamp(start_ts).replace( + tzinfo=user.profile.tzinfo + ) + # TODO Add a shim here temporarily to fix imports while we were in France + # if date is between 10/15 and 12/15, cast it to Europe/Central + if ( + datetime(2023, 10, 15).replace( + tzinfo=pytz.timezone("Europe/Paris") + ) + <= timestamp + <= datetime(2023, 12, 15).replace( + tzinfo=pytz.timezone("Europe/Paris") + ) + ): + timestamp.replace(tzinfo=pytz.timezone("Europe/Paris")) scrobble = Scrobble.objects.filter( timestamp=timestamp, @@ -235,10 +260,19 @@ def build_scrobbles_from_book_map(book_map: dict, user: "User") -> list["Scrobbl user_id=user.id, ).first() if scrobble: - logger.info(f"Found existing scrobble {scrobble}, updating") + logger.info( + f"Found existing scrobble {scrobble}, updating" + ) scrobble.book_page_data = scrobble_page_data - scrobble.playback_position_seconds = scrobble.calc_reading_duration() - scrobble.save(update_fields=["book_page_data", "playback_position_seconds"]) + scrobble.playback_position_seconds = ( + scrobble.calc_reading_duration() + ) + scrobble.save( + update_fields=[ + "book_page_data", + "playback_position_seconds", + ] + ) if not scrobble: logger.info( f"Queueing scrobble for {book_id}, page {page_number}" @@ -275,15 +309,20 @@ def fix_long_play_stats_for_scrobbles(scrobbles: list) -> None: # But if there's a next scrobble, set pages read to their starting page # if scrobble.previous: - scrobble.long_play_seconds = scrobble.playback_position_seconds + scrobble.previous.long_play_seconds - scrobble.book_pages_read = scrobble.book_pages_read + scrobble.previous.book_pages_read + scrobble.long_play_seconds = scrobble.playback_position_seconds + ( + scrobble.previous.long_play_seconds or 0 + ) + if not scrobble.book_pages_read: + scrobble.book_pages_read = ( + scrobble.calc_pages_read() + + scrobble.previous.book_pages_read + ) else: scrobble.long_play_seconds = scrobble.playback_position_seconds scrobble.book_pages_read = scrobble.calc_pages_read() - scrobble.save( - update_fields=["book_pages_read", "long_play_seconds"] - ) + scrobble.save(update_fields=["book_pages_read", "long_play_seconds"]) + def process_koreader_sqlite_file(file_path, user_id) -> list: """Given a sqlite file from KoReader, open the book table, iterate @@ -301,7 +340,9 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: con = sqlite3.connect(file_path) cur = con.cursor() book_map = build_book_map(cur.execute("SELECT * FROM book")) - book_map = build_page_data(cur.execute("SELECT * from page_stat_data"), book_map, tz) + book_map = build_page_data( + cur.execute("SELECT * from page_stat_data"), book_map, tz + ) new_scrobbles = build_scrobbles_from_book_map(book_map, user) else: for table_name, pragma_table_info, rows in stream_sqlite( diff --git a/vrobbler/apps/books/models.py b/vrobbler/apps/books/models.py index 1a4183d..e46fb1c 100644 --- a/vrobbler/apps/books/models.py +++ b/vrobbler/apps/books/models.py @@ -185,6 +185,12 @@ class Book(LongPlayScrobblableMixin): if data.get("pages") == None: data.pop("pages") + if not isinstance(data.get("pages"), int): + logger.info( + f"Pages for {self} from OL expected to be int, but got {data.get('pages')}" + ) + data.pop("pages") + # Pop this, so we can look it up later cover_url = data.pop("cover_url", "")