diff --git a/vrobbler/apps/books/koreader.py b/vrobbler/apps/books/koreader.py index d315a37..c62213f 100644 --- a/vrobbler/apps/books/koreader.py +++ b/vrobbler/apps/books/koreader.py @@ -93,7 +93,7 @@ def create_book_from_row(row: list): title=row[KoReaderBookColumn.TITLE.value], pages=total_pages, koreader_data_by_hash={ - row[KoReaderBookColumn.MD5.value]: { + str(row[KoReaderBookColumn.MD5.value]): { "title": row[KoReaderBookColumn.TITLE.value], "author_str": author_str, "book_id": row[KoReaderBookColumn.ID.value], @@ -233,7 +233,7 @@ def build_scrobbles_from_book_map( should_create_scrobble = True if should_create_scrobble: - scrobble_page_data = OrderedDict( + scrobble_page_data = dict( sorted( scrobble_page_data.items(), key=lambda x: x[1]["start_ts"], @@ -359,6 +359,7 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: is_os_file = "https://" not in file_path if is_os_file: + # Loading sqlite file from local filesystem con = sqlite3.connect(file_path) cur = con.cursor() try: @@ -376,6 +377,8 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: ) new_scrobbles = build_scrobbles_from_book_map(book_map, user) else: + # Streaming the sqlite file off S3 + book_map = {} for table_name, pragma_table_info, rows in stream_sqlite( _sqlite_bytes(file_path), max_buffer_size=1_048_576 ): @@ -383,6 +386,9 @@ def process_koreader_sqlite_file(file_path, user_id) -> list: if table_name == "book": book_map = build_book_map(rows) + for table_name, pragma_table_info, rows in stream_sqlite( + _sqlite_bytes(file_path), max_buffer_size=1_048_576 + ): if table_name == "page_stat_data": book_map = build_page_data(rows, book_map, tz) new_scrobbles = build_scrobbles_from_book_map(book_map, user) diff --git a/vrobbler/apps/books/models.py b/vrobbler/apps/books/models.py index 7c724d4..53bdcc7 100644 --- a/vrobbler/apps/books/models.py +++ b/vrobbler/apps/books/models.py @@ -34,6 +34,7 @@ from vrobbler.apps.books.locg import ( lookup_comic_from_locg, lookup_comic_writer_by_locg_slug, ) +from vrobbler.apps.scrobbles.dataclasses import BookLogData COMICVINE_API_KEY = getattr(settings, "COMICVINE_API_KEY", "") @@ -137,6 +138,10 @@ class Book(LongPlayScrobblableMixin): def subtitle(self): return f" by {self.author}" + @property + def logdata_cls(self): + return BookLogData + @property def primary_image_url(self) -> str: url = "" @@ -278,19 +283,24 @@ class Book(LongPlayScrobblableMixin): author.headshot.save(fname, ContentFile(r.content), save=True) self.authors.add(author) - - def page_data_for_user(self, user_id: int, convert_timestamps: bool=True) -> dict: + def page_data_for_user( + self, user_id: int, convert_timestamps: bool = True + ) -> dict: scrobbles = self.scrobble_set.filter(user=user_id) pages = {} for scrobble in scrobbles: - if scrobble.book_page_data: - for page, data in scrobble.book_page_data.items(): + if scrobble.logdata.page_data: + for page, data in scrobble.logdata.page_data.items(): if convert_timestamps: - data["start_ts"] = datetime.fromtimestamp(data["start_ts"]) + data["start_ts"] = datetime.fromtimestamp( + data["start_ts"] + ) data["end_ts"] = datetime.fromtimestamp(data["end_ts"]) pages[page] = data - sorted_pages = OrderedDict(sorted(pages.items(), key=lambda x: x[1]["start_ts"])) + sorted_pages = OrderedDict( + sorted(pages.items(), key=lambda x: x[1]["start_ts"]) + ) return sorted_pages diff --git a/vrobbler/apps/books/tests/conftest.py b/vrobbler/apps/books/tests/conftest.py index 8e47f34..fb87337 100644 --- a/vrobbler/apps/books/tests/conftest.py +++ b/vrobbler/apps/books/tests/conftest.py @@ -48,7 +48,7 @@ class KoReaderBookRows: 300 + wiggle, self.DEFAULT_STR, self.DEFAULT_STR, - hashlib.md5(title.encode()), + hashlib.md5(title.encode()).hexdigest(), i * wiggle * 20, 120, ] diff --git a/vrobbler/apps/books/tests/test_koreader.py b/vrobbler/apps/books/tests/test_koreader.py index 2cfcd61..f55b5a6 100644 --- a/vrobbler/apps/books/tests/test_koreader.py +++ b/vrobbler/apps/books/tests/test_koreader.py @@ -21,9 +21,10 @@ def test_build_book_map(get_mock, koreader_rows, valid_response): @mock.patch("requests.get") def test_load_page_data_to_map(get_mock, koreader_rows, valid_response): get_mock.return_value = valid_response - book_map = build_book_map(koreader_rows.BOOK_ROWS) - - book_map = build_page_data(koreader_rows.PAGE_STATS_ROWS, book_map) + book_map = build_page_data( + koreader_rows.PAGE_STATS_ROWS, + build_book_map(koreader_rows.BOOK_ROWS), + ) assert ( len(book_map[1]["pages"]) == koreader_rows.BOOK_ROWS[0][ @@ -45,9 +46,9 @@ def test_build_scrobbles_from_pages( # Corresponds to number of sessions per book ( 20 pages per session, 120 +/- 15 pages read ) expected_scrobbles = 6 * len(book_map.keys()) assert len(scrobbles) == expected_scrobbles - assert len(scrobbles[0].book_page_data.keys()) == 22 - assert len(scrobbles[1].book_page_data.keys()) == 20 - assert len(scrobbles[2].book_page_data.keys()) == 20 - assert len(scrobbles[3].book_page_data.keys()) == 20 - assert len(scrobbles[4].book_page_data.keys()) == 20 - assert len(scrobbles[5].book_page_data.keys()) == 18 + assert len(scrobbles[0].logdata.page_data.keys()) == 21 + assert len(scrobbles[1].logdata.page_data.keys()) == 20 + assert len(scrobbles[2].logdata.page_data.keys()) == 20 + assert len(scrobbles[3].logdata.page_data.keys()) == 20 + assert len(scrobbles[4].logdata.page_data.keys()) == 20 + assert len(scrobbles[5].logdata.page_data.keys()) == 18 diff --git a/vrobbler/apps/scrobbles/dataclasses.py b/vrobbler/apps/scrobbles/dataclasses.py index 56f5c5f..8463805 100644 --- a/vrobbler/apps/scrobbles/dataclasses.py +++ b/vrobbler/apps/scrobbles/dataclasses.py @@ -123,13 +123,12 @@ class BookPageLogData(JSONDataclass): @dataclass class BookLogData(LongPlayLogData): - serial_scrobble_id: Optional[int] - long_play_complete: bool = False + long_play_complete: Optional[bool] = None koreader_hash: Optional[str] = None + page_data: Optional[dict[int, BookPageLogData]] = None pages_read: Optional[int] = None - page_data: Optional[list[BookPageLogData]] = None - page_end: Optional[int] = None page_start: Optional[int] = None + page_end: Optional[int] = None serial_scrobble_id: Optional[int] = None