[version] Bump to version 0.15.3

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vrobbler"
-version = "0.15.0"
+version = "0.15.3"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 

vrobbler/apps/books/koreader.py

@@ -93,7 +93,7 @@ def create_book_from_row(row: list):
         title=row[KoReaderBookColumn.TITLE.value],
         pages=total_pages,
         koreader_data_by_hash={
-            row[KoReaderBookColumn.MD5.value]: {
+            str(row[KoReaderBookColumn.MD5.value]): {
                 "title": row[KoReaderBookColumn.TITLE.value],
                 "author_str": author_str,
                 "book_id": row[KoReaderBookColumn.ID.value],
@@ -233,7 +233,7 @@ def build_scrobbles_from_book_map(
                 should_create_scrobble = True
 
             if should_create_scrobble:
-                scrobble_page_data = OrderedDict(
+                scrobble_page_data = dict(
                     sorted(
                         scrobble_page_data.items(),
                         key=lambda x: x[1]["start_ts"],
@@ -359,6 +359,7 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
 
     is_os_file = "https://" not in file_path
     if is_os_file:
+        # Loading sqlite file from local filesystem
         con = sqlite3.connect(file_path)
         cur = con.cursor()
         try:
@@ -376,6 +377,8 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
         )
         new_scrobbles = build_scrobbles_from_book_map(book_map, user)
     else:
+        # Streaming the sqlite file off S3
+        book_map = {}
         for table_name, pragma_table_info, rows in stream_sqlite(
             _sqlite_bytes(file_path), max_buffer_size=1_048_576
         ):
@@ -383,6 +386,9 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
             if table_name == "book":
                 book_map = build_book_map(rows)
 
+        for table_name, pragma_table_info, rows in stream_sqlite(
+            _sqlite_bytes(file_path), max_buffer_size=1_048_576
+        ):
             if table_name == "page_stat_data":
                 book_map = build_page_data(rows, book_map, tz)
                 new_scrobbles = build_scrobbles_from_book_map(book_map, user)

vrobbler/apps/books/models.py

@@ -34,6 +34,7 @@ from vrobbler.apps.books.locg import (
     lookup_comic_from_locg,
     lookup_comic_writer_by_locg_slug,
 )
+from vrobbler.apps.scrobbles.dataclasses import BookLogData
 
 COMICVINE_API_KEY = getattr(settings, "COMICVINE_API_KEY", "")
 
@@ -137,6 +138,10 @@ class Book(LongPlayScrobblableMixin):
     def subtitle(self):
         return f" by {self.author}"
 
+    @property
+    def logdata_cls(self):
+        return BookLogData
+
     @property
     def primary_image_url(self) -> str:
         url = ""
@@ -278,19 +283,24 @@ class Book(LongPlayScrobblableMixin):
                 author.headshot.save(fname, ContentFile(r.content), save=True)
         self.authors.add(author)
 
-
-    def page_data_for_user(self, user_id: int, convert_timestamps: bool=True) -> dict:
+    def page_data_for_user(
+        self, user_id: int, convert_timestamps: bool = True
+    ) -> dict:
         scrobbles = self.scrobble_set.filter(user=user_id)
 
         pages = {}
         for scrobble in scrobbles:
-            if scrobble.book_page_data:
-                for page, data in scrobble.book_page_data.items():
+            if scrobble.logdata.page_data:
+                for page, data in scrobble.logdata.page_data.items():
                     if convert_timestamps:
-                        data["start_ts"] = datetime.fromtimestamp(data["start_ts"])
+                        data["start_ts"] = datetime.fromtimestamp(
+                            data["start_ts"]
+                        )
                         data["end_ts"] = datetime.fromtimestamp(data["end_ts"])
                     pages[page] = data
-        sorted_pages = OrderedDict(sorted(pages.items(), key=lambda x: x[1]["start_ts"]))
+        sorted_pages = OrderedDict(
+            sorted(pages.items(), key=lambda x: x[1]["start_ts"])
+        )
 
         return sorted_pages
 

vrobbler/apps/books/tests/conftest.py

@@ -48,7 +48,7 @@ class KoReaderBookRows:
             300 + wiggle,
             self.DEFAULT_STR,
             self.DEFAULT_STR,
-            hashlib.md5(title.encode()),
+            hashlib.md5(title.encode()).hexdigest(),
             i * wiggle * 20,
             120,
         ]

vrobbler/apps/books/tests/test_koreader.py

@@ -21,9 +21,10 @@ def test_build_book_map(get_mock, koreader_rows, valid_response):
 @mock.patch("requests.get")
 def test_load_page_data_to_map(get_mock, koreader_rows, valid_response):
     get_mock.return_value = valid_response
-    book_map = build_book_map(koreader_rows.BOOK_ROWS)
-
-    book_map = build_page_data(koreader_rows.PAGE_STATS_ROWS, book_map)
+    book_map = build_page_data(
+        koreader_rows.PAGE_STATS_ROWS,
+        build_book_map(koreader_rows.BOOK_ROWS),
+    )
     assert (
         len(book_map[1]["pages"])
         == koreader_rows.BOOK_ROWS[0][
@@ -45,9 +46,9 @@ def test_build_scrobbles_from_pages(
     # Corresponds to number of sessions per book ( 20 pages per session, 120 +/- 15 pages read )
     expected_scrobbles = 6 * len(book_map.keys())
     assert len(scrobbles) == expected_scrobbles
-    assert len(scrobbles[0].book_page_data.keys()) == 22
-    assert len(scrobbles[1].book_page_data.keys()) == 20
-    assert len(scrobbles[2].book_page_data.keys()) == 20
-    assert len(scrobbles[3].book_page_data.keys()) == 20
-    assert len(scrobbles[4].book_page_data.keys()) == 20
-    assert len(scrobbles[5].book_page_data.keys()) == 18
+    assert len(scrobbles[0].logdata.page_data.keys()) == 21
+    assert len(scrobbles[1].logdata.page_data.keys()) == 20
+    assert len(scrobbles[2].logdata.page_data.keys()) == 20
+    assert len(scrobbles[3].logdata.page_data.keys()) == 20
+    assert len(scrobbles[4].logdata.page_data.keys()) == 20
+    assert len(scrobbles[5].logdata.page_data.keys()) == 18

vrobbler/apps/scrobbles/dataclasses.py

@@ -30,9 +30,34 @@ class JSONDataclass(JSONWizard):
     def json(self):
         return json.dumps(self.asdict)
 
+
 class LongPlayLogData(JSONDataclass):
     serial_scrobble_id: Optional[int]
-    long_play_complete: Optional[bool]
+    long_play_complete: bool = False
+
+
+class WithOthersLogData(JSONDataclass):
+    with_user_ids: Optional[list[int]] = None
+    with_names_str: Optional[list[str]] = None
+
+    @property
+    def with_names(self) -> list[str]:
+        with_names = []
+        if self.with_user_ids:
+            with_names += [u.full_name for u in self.with_users if u]
+        if self.with_names_str:
+            with_names += [u for u in self.with_names_str]
+        return with_names
+
+    @property
+    def with_users(self) -> list[User]:
+        with_users = []
+        if self.with_user_ids:
+            with_users = [
+                User.objects.filter(id=i).first() for i in self.with_user_ids
+            ]
+        return with_users
+
 
 @dataclass
 class BoardGameScoreLogData(JSONDataclass):
@@ -73,6 +98,8 @@ class BoardGameScoreLogData(JSONDataclass):
 
 @dataclass
 class BoardGameLogData(LongPlayLogData):
+    serial_scrobble_id: Optional[int]
+    long_play_complete: bool = False
     players: Optional[list[BoardGameScoreLogData]] = None
     location: Optional[str] = None
     geo_location_id: Optional[int] = None
@@ -96,45 +123,30 @@ class BookPageLogData(JSONDataclass):
 
 @dataclass
 class BookLogData(LongPlayLogData):
-    koreader_hash: Optional[str]
-    pages_read: Optional[int]
-    page_data: Optional[list[BookPageLogData]]
-    page_end: Optional[int]
-    page_start: Optional[int]
-    serial_scrobble_id: Optional[int]
+    long_play_complete: Optional[bool] = None
+    koreader_hash: Optional[str] = None
+    page_data: Optional[dict[int, BookPageLogData]] = None
+    pages_read: Optional[int] = None
+    page_start: Optional[int] = None
+    page_end: Optional[int] = None
+    serial_scrobble_id: Optional[int] = None
 
 
 @dataclass
-class LifeEventLogData(JSONDataclass):
-    participant_user_ids: Optional[list[int]] = None
-    participant_names: Optional[list[str]] = None
+class LifeEventLogData(WithOthersLogData):
+    with_user_ids: Optional[list[int]] = None
+    with_names_str: Optional[list[str]] = None
     location: Optional[str] = None
     geo_location_id: Optional[int] = None
     details: Optional[str] = None
 
-    def participants(self) -> list[str]:
-        participants = []
-        if self.participant_user_ids:
-            participants += [u.full_name for u in self.participant_users]
-        if self.participant_names:
-            participants += self.participant_names
-        return participants
-
-    @property
-    def participant_users(self) -> list[User]:
-        participants = []
-        if self.participant_user_ids:
-            for id in self.participant_user_ids:
-                participants.append(User.objects.filter(id=id).first())
-        return participants
-
     def geo_location(self):
         return GeoLocation.objects.filter(id=self.geo_location_id).first()
 
 
 @dataclass
 class MoodLogData(JSONDataclass):
-    reasons: Optional[str]
+    reasons: Optional[str] = None
 
 
 @dataclass
@@ -144,23 +156,27 @@ class VideoLogData(JSONDataclass):
     run_time_seconds: int
     kind: str
     year: Optional[int]
-    episode_number: Optional[int]
-    source_url: Optional[str]
-    imdbID: Optional[str]
-    season_number: Optional[int]
-    cover_url = Optional[str]
-    next_imdb_id: Optional[int]
-    tv_series_id: Optional[str]
+    episode_number: Optional[int] = None
+    source_url: Optional[str] = None
+    imdbID: Optional[str] = None
+    season_number: Optional[int] = None
+    cover_url: Optional[str] = None
+    next_imdb_id: Optional[int] = None
+    tv_series_id: Optional[str] = None
 
 
 @dataclass
 class VideoGameLogData(LongPlayLogData):
+    serial_scrobble_id: Optional[int]
+    long_play_complete: bool = False
+    console: str = ""
     emulated: bool = False
-    console: Optional[str] = None
     emulator: Optional[str] = None
 
 
 @dataclass
-class BrickSetLogData(LongPlayLogData):
-    built_with_names: Optional[list[str]]
-    built_with_user_ids: Optional[list[str]]
+class BrickSetLogData(LongPlayLogData, WithOthersLogData):
+    serial_scrobble_id: Optional[int]
+    long_play_complete: bool = False
+    with_user_ids: Optional[list[int]] = None
+    with_names_str: Optional[list[str]] = None

vrobbler/apps/scrobbles/models.py

@@ -512,7 +512,9 @@ class Scrobble(TimeStampedModel):
         LifeEvent, on_delete=models.DO_NOTHING, **BNULL
     )
     mood = models.ForeignKey(Mood, on_delete=models.DO_NOTHING, **BNULL)
-    brickset = models.ForeignKey(BrickSet, on_delete=models.DO_NOTHING, **BNULL)
+    brickset = models.ForeignKey(
+        BrickSet, on_delete=models.DO_NOTHING, **BNULL
+    )
     media_type = models.CharField(
         max_length=14, choices=MediaType.choices, default=MediaType.VIDEO
     )
@@ -894,15 +896,15 @@ class Scrobble(TimeStampedModel):
 
     def calc_reading_duration(self) -> int:
         duration = 0
-        if self.book_page_data:
-            for k, v in self.book_page_data.items():
+        if self.logdata.page_data:
+            for k, v in self.logdata.page_data.items():
                 duration += v.get("duration")
         return duration
 
     def calc_pages_read(self) -> int:
         pages_read = 0
-        if self.book_page_data:
-            pages = [int(k) for k in self.book_page_data.keys()]
+        if self.logdata.page_data:
+            pages = [int(k) for k in self.logdata.page_data.keys()]
             pages.sort()
             if len(pages) == 1:
                 pages_read = 1
@@ -915,8 +917,8 @@ class Scrobble(TimeStampedModel):
     @property
     def last_page_read(self) -> int:
         last_page = 0
-        if self.book_page_data:
-            pages = [int(k) for k in self.book_page_data.keys()]
+        if self.logdata.page_data:
+            pages = [int(k) for k in self.logdata.page_data.keys()]
             pages.sort()
             last_page = pages[-1]
         return last_page

vrobbler/apps/videos/utils.py

@@ -45,8 +45,6 @@ def get_or_create_video(data_dict: dict, post_keys: dict, force_update=False):
 
         series_name = video_dict.pop("series_name", None)
         if series_name:
-
-            series_name = video_dict.pop("series_name")
             series, series_created = Series.objects.get_or_create(
                 name=series_name
             )