diff --git a/PROJECT.org b/PROJECT.org index 40d04c0..669e26b 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [1/16] :vrobbler:project:personal: +* Backlog [2/17] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -544,6 +544,26 @@ log a warning and move on. We should have a global view `/favorites/` that shows the logged in users's favorited media objects. +** DONE [#A] Fix book scrobbles where page_data is a list :bug:books:scrobbles: +:PROPERTIES: +:ID: 35b323fa-ccc0-4009-b227-8a0f12bbd469 +:END: + +*** Description + +Comic scrobbling is currently kind of janky. Most of the problems boil down to them +storing saved page data in a list of dicts rather than a dict keyed off of the page number that +was read. + +We need to adjust comic scrobbling to use a dict of pages keyed off the page number, and also +write a migration script that runs as a data migration to update any book scrobbles that may +have page_data as a list. + +*** Example data +#+begin_src python +{"notes": null, "page_end": 27, "page_data": [{"notes": null, "end_ts": 1771815895, "duration": 14, "start_ts": 1771815881, "description": null, "page_number": "1"}, {"notes": null, "end_ts": 1771815908, "duration": 13, "start_ts": 1771815895, "description": null, "page_number": "1"}, {"notes": null, "end_ts": 1771815913, "duration": 5, "start_ts": 1771815908, "description": null, "page_number": "2"}, {"notes": null, "end_ts": 1771815933, "duration": 20, "start_ts": 1771815913, "description": null, "page_number": "3"}, {"notes": null, "end_ts": 1771815945, "duration": 12, "start_ts": 1771815933, "description": null, "page_number": "4"}, {"notes": null, "end_ts": 1771815983, "duration": 38, "start_ts": 1771815945, "description": null, "page_number": "5"}, {"notes": null, "end_ts": 1771816007, "duration": 24, "start_ts": 1771815983, "description": null, "page_number": "6"}, {"notes": null, "end_ts": 1771816011, "duration": 4, "start_ts": 1771816007, "description": null, "page_number": "7"}, {"notes": null, "end_ts": 1771816013, "duration": 2, "start_ts": 1771816011, "description": null, "page_number": "8"}, {"notes": null, "end_ts": 1771816052, "duration": 39, "start_ts": 1771816013, "description": null, "page_number": "7"}, {"notes": null, "end_ts": 1771816127, "duration": 75, "start_ts": 1771816052, "description": null, "page_number": "8"}, {"notes": null, "end_ts": 1771816134, "duration": 7, "start_ts": 1771816127, "description": null, "page_number": "9"}, {"notes": null, "end_ts": 1771816196, "duration": 62, "start_ts": 1771816134, "description": null, "page_number": "10"}, {"notes": null, "end_ts": 1771816262, "duration": 66, "start_ts": 1771816196, "description": null, "page_number": "11"}, {"notes": null, "end_ts": 1771816293, "duration": 31, "start_ts": 1771816262, "description": null, "page_number": "12"}, {"notes": null, "end_ts": 1771816322, "duration": 29, "start_ts": 1771816293, "description": null, "page_number": "13"}, {"notes": null, "end_ts": 1771816330, "duration": 8, "start_ts": 1771816322, "description": null, "page_number": "14"}, {"notes": null, "end_ts": 1771816368, "duration": 38, "start_ts": 1771816330, "description": null, "page_number": "15"}, {"notes": null, "end_ts": 1771816388, "duration": 20, "start_ts": 1771816368, "description": null, "page_number": "16"}, {"notes": null, "end_ts": 1771816482, "duration": 94, "start_ts": 1771816388, "description": null, "page_number": "17"}, {"notes": null, "end_ts": 1771816550, "duration": 68, "start_ts": 1771816482, "description": null, "page_number": "18"}, {"notes": null, "end_ts": 1771816567, "duration": 17, "start_ts": 1771816550, "description": null, "page_number": "19"}, {"notes": null, "end_ts": 1771816586, "duration": 19, "start_ts": 1771816567, "description": null, "page_number": "20"}, {"notes": null, "end_ts": 1771816597, "duration": 11, "start_ts": 1771816586, "description": null, "page_number": "21"}, {"notes": null, "end_ts": 1771816616, "duration": 19, "start_ts": 1771816597, "description": null, "page_number": "22"}, {"notes": null, "end_ts": 1771816640, "duration": 24, "start_ts": 1771816616, "description": null, "page_number": "23"}, {"notes": null, "end_ts": 1771816690, "duration": 50, "start_ts": 1771816640, "description": null, "page_number": "24"}, {"notes": null, "end_ts": 1771816702, "duration": 12, "start_ts": 1771816690, "description": null, "page_number": "25"}, {"notes": null, "end_ts": 1771816823, "duration": 121, "start_ts": 1771816702, "description": null, "page_number": "26"}, {"notes": null, "end_ts": null, "duration": null, "start_ts": 1771816823, "description": null, "page_number": "27"}], "page_start": 1, "pages_read": 27, "resume_url": null, "description": null, "koreader_hash": null, "long_play_complete": false} + +#+end_src ** DONE [#A] Lichess imports do not set default visbility :boardgames:bug:importers:lichess: :PROPERTIES: :ID: a78f7c72-a20a-8db2-cde0-d92a731d4fba diff --git a/vrobbler/apps/scrobbles/migrations/0096_convert_book_page_data_to_dict.py b/vrobbler/apps/scrobbles/migrations/0096_convert_book_page_data_to_dict.py new file mode 100644 index 0000000..8ef0b8d --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0096_convert_book_page_data_to_dict.py @@ -0,0 +1,35 @@ +from django.db import migrations + + +def convert_page_data_to_dict(apps, schema_editor): + Scrobble = apps.get_model("scrobbles", "Scrobble") + for scrobble in Scrobble.objects.filter(media_type="Book").exclude( + log__page_data=None + ): + page_data = scrobble.log.get("page_data") + if isinstance(page_data, list): + new_page_data = {} + for entry in page_data: + page_num = entry.get("page_number") + if page_num is not None: + try: + page_num = int(page_num) + except (ValueError, TypeError): + continue + new_page_data[page_num] = entry + scrobble.log["page_data"] = new_page_data + scrobble.save(update_fields=["log"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ("scrobbles", "0095_backfill_null_visibility"), + ] + + operations = [ + migrations.RunPython( + convert_page_data_to_dict, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index cb52d61..e568a83 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -1412,20 +1412,18 @@ class Scrobble(TimeStampedModel): or scrobble_data["playback_status"] == "stopped" ): if read_log_page: - page_list = scrobble.log.get("page_data", []) - if page_list: - for page in page_list: + page_data = scrobble.log.get("page_data", {}) + if page_data: + for page_num, page in page_data.items(): if not page.get("end_ts", None): page["end_ts"] = int(timezone.now().timestamp()) page["duration"] = page["end_ts"] - page.get("start_ts") - page_list.append( - BookPageLogData( - page_number=read_log_page, - start_ts=int(timezone.now().timestamp()), - ) + page_data[read_log_page] = BookPageLogData( + page_number=read_log_page, + start_ts=int(timezone.now().timestamp()), ) - scrobble.log["page_data"] = page_list + scrobble.log["page_data"] = page_data scrobble.save(update_fields=["log"]) elif "log" in scrobble_data.keys() and scrobble.log: scrobble_data["log"] = scrobble.log | scrobble_data["log"] @@ -1436,12 +1434,12 @@ class Scrobble(TimeStampedModel): if read_log_page: scrobble_data["log"] = BookLogData( - page_data=[ - BookPageLogData( + page_data={ + read_log_page: BookPageLogData( page_number=read_log_page, start_ts=int(timezone.now().timestamp()), ) - ] + } ) logger.info( @@ -1783,30 +1781,24 @@ class Scrobble(TimeStampedModel): page_data = self.log.get("page_data") if page_data: - # --- Sort safely by numeric page_number --- - def safe_page_number(entry): - try: - return int(getattr("page_number", entry), 0) - except (ValueError, TypeError): - return float("inf") - if isinstance(page_data, dict): - logger.warning("Page data is dict, migrate koreader data") - return + valid_pages = sorted(int(k) for k in page_data.keys()) + if valid_pages: + self.log["page_start"] = min(valid_pages) + self.log["page_end"] = max(valid_pages) + self.log["pages_read"] = len(set(valid_pages)) + elif isinstance(page_data, list): + valid_pages = [] + for page in page_data: + try: + valid_pages.append(int(page["page_number"])) + except (ValueError, TypeError): + continue - page_data.sort(key=safe_page_number) - - valid_pages = [] - for page in page_data: - try: - valid_pages.append(int(page["page_number"])) - except (ValueError, TypeError): - continue - - if valid_pages: - self.log["page_start"] = min(valid_pages) - self.log["page_end"] = max(valid_pages) - self.log["pages_read"] = len(set(valid_pages)) + if valid_pages: + self.log["page_start"] = min(valid_pages) + self.log["page_end"] = max(valid_pages) + self.log["pages_read"] = len(set(valid_pages)) else: page_start = self.log.get("page_start") page_end = self.log.get("page_end")