secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[books] Fix list page_data for old book scrobbles
All checks were successful
build / test (push) Successful in 2m2s
Details

Browse Source
This commit is contained in:
Colin Powell
2026-06-11 11:00:04 -04:00
parent c16b61db40
commit a171192a6f
3 changed files with 82 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
PROJECT.org
View File

@ -88,7 +88,7 @@ fetching and simple saving.
*** Metadata sources
**** Scraper
* Backlog [1/16] :vrobbler:project:personal:
* Backlog [2/17] :vrobbler:project:personal:
** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles:
:PROPERTIES:
:ID: 702462cf-d54b-48c6-8a7c-78b8de751deb
@ -544,6 +544,26 @@ log a warning and move on.
We should have a global view `/favorites/` that shows the logged in users's
favorited media objects.
** DONE [#A] Fix book scrobbles where page_data is a list :bug:books:scrobbles:
:PROPERTIES:
:ID: 35b323fa-ccc0-4009-b227-8a0f12bbd469
:END:
*** Description
Comic scrobbling is currently kind of janky. Most of the problems boil down to them
storing saved page data in a list of dicts rather than a dict keyed off of the page number that
was read.
We need to adjust comic scrobbling to use a dict of pages keyed off the page number, and also
write a migration script that runs as a data migration to update any book scrobbles that may
have page_data as a list.
*** Example data
#+begin_src python
{"notes": null, "page_end": 27, "page_data": [{"notes": null, "end_ts": 1771815895, "duration": 14, "start_ts": 1771815881, "description": null, "page_number": "1"}, {"notes": null, "end_ts": 1771815908, "duration": 13, "start_ts": 1771815895, "description": null, "page_number": "1"}, {"notes": null, "end_ts": 1771815913, "duration": 5, "start_ts": 1771815908, "description": null, "page_number": "2"}, {"notes": null, "end_ts": 1771815933, "duration": 20, "start_ts": 1771815913, "description": null, "page_number": "3"}, {"notes": null, "end_ts": 1771815945, "duration": 12, "start_ts": 1771815933, "description": null, "page_number": "4"}, {"notes": null, "end_ts": 1771815983, "duration": 38, "start_ts": 1771815945, "description": null, "page_number": "5"}, {"notes": null, "end_ts": 1771816007, "duration": 24, "start_ts": 1771815983, "description": null, "page_number": "6"}, {"notes": null, "end_ts": 1771816011, "duration": 4, "start_ts": 1771816007, "description": null, "page_number": "7"}, {"notes": null, "end_ts": 1771816013, "duration": 2, "start_ts": 1771816011, "description": null, "page_number": "8"}, {"notes": null, "end_ts": 1771816052, "duration": 39, "start_ts": 1771816013, "description": null, "page_number": "7"}, {"notes": null, "end_ts": 1771816127, "duration": 75, "start_ts": 1771816052, "description": null, "page_number": "8"}, {"notes": null, "end_ts": 1771816134, "duration": 7, "start_ts": 1771816127, "description": null, "page_number": "9"}, {"notes": null, "end_ts": 1771816196, "duration": 62, "start_ts": 1771816134, "description": null, "page_number": "10"}, {"notes": null, "end_ts": 1771816262, "duration": 66, "start_ts": 1771816196, "description": null, "page_number": "11"}, {"notes": null, "end_ts": 1771816293, "duration": 31, "start_ts": 1771816262, "description": null, "page_number": "12"}, {"notes": null, "end_ts": 1771816322, "duration": 29, "start_ts": 1771816293, "description": null, "page_number": "13"}, {"notes": null, "end_ts": 1771816330, "duration": 8, "start_ts": 1771816322, "description": null, "page_number": "14"}, {"notes": null, "end_ts": 1771816368, "duration": 38, "start_ts": 1771816330, "description": null, "page_number": "15"}, {"notes": null, "end_ts": 1771816388, "duration": 20, "start_ts": 1771816368, "description": null, "page_number": "16"}, {"notes": null, "end_ts": 1771816482, "duration": 94, "start_ts": 1771816388, "description": null, "page_number": "17"}, {"notes": null, "end_ts": 1771816550, "duration": 68, "start_ts": 1771816482, "description": null, "page_number": "18"}, {"notes": null, "end_ts": 1771816567, "duration": 17, "start_ts": 1771816550, "description": null, "page_number": "19"}, {"notes": null, "end_ts": 1771816586, "duration": 19, "start_ts": 1771816567, "description": null, "page_number": "20"}, {"notes": null, "end_ts": 1771816597, "duration": 11, "start_ts": 1771816586, "description": null, "page_number": "21"}, {"notes": null, "end_ts": 1771816616, "duration": 19, "start_ts": 1771816597, "description": null, "page_number": "22"}, {"notes": null, "end_ts": 1771816640, "duration": 24, "start_ts": 1771816616, "description": null, "page_number": "23"}, {"notes": null, "end_ts": 1771816690, "duration": 50, "start_ts": 1771816640, "description": null, "page_number": "24"}, {"notes": null, "end_ts": 1771816702, "duration": 12, "start_ts": 1771816690, "description": null, "page_number": "25"}, {"notes": null, "end_ts": 1771816823, "duration": 121, "start_ts": 1771816702, "description": null, "page_number": "26"}, {"notes": null, "end_ts": null, "duration": null, "start_ts": 1771816823, "description": null, "page_number": "27"}], "page_start": 1, "pages_read": 27, "resume_url": null, "description": null, "koreader_hash": null, "long_play_complete": false}
#+end_src
** DONE [#A] Lichess imports do not set default visbility :boardgames:bug:importers:lichess:
:PROPERTIES:
:ID: a78f7c72-a20a-8db2-cde0-d92a731d4fba

35
vrobbler/apps/scrobbles/migrations/0096_convert_book_page_data_to_dict.py Normal file
View File

@ -0,0 +1,35 @@
from django.db import migrations
def convert_page_data_to_dict(apps, schema_editor):
Scrobble = apps.get_model("scrobbles", "Scrobble")
for scrobble in Scrobble.objects.filter(media_type="Book").exclude(
log__page_data=None
):
page_data = scrobble.log.get("page_data")
if isinstance(page_data, list):
new_page_data = {}
for entry in page_data:
page_num = entry.get("page_number")
if page_num is not None:
try:
page_num = int(page_num)
except (ValueError, TypeError):
continue
new_page_data[page_num] = entry
scrobble.log["page_data"] = new_page_data
scrobble.save(update_fields=["log"])
class Migration(migrations.Migration):
dependencies = [
("scrobbles", "0095_backfill_null_visibility"),
]
operations = [
migrations.RunPython(
convert_page_data_to_dict,
reverse_code=migrations.RunPython.noop,
),
]

60
vrobbler/apps/scrobbles/models.py
View File

@ -1412,20 +1412,18 @@ class Scrobble(TimeStampedModel):
or scrobble_data["playback_status"] == "stopped"
):
if read_log_page:
page_list = scrobble.log.get("page_data", [])
if page_list:
for page in page_list:
page_data = scrobble.log.get("page_data", {})
if page_data:
for page_num, page in page_data.items():
if not page.get("end_ts", None):
page["end_ts"] = int(timezone.now().timestamp())
page["duration"] = page["end_ts"] - page.get("start_ts")
page_list.append(
BookPageLogData(
page_number=read_log_page,
start_ts=int(timezone.now().timestamp()),
)
page_data[read_log_page] = BookPageLogData(
page_number=read_log_page,
start_ts=int(timezone.now().timestamp()),
)
scrobble.log["page_data"] = page_list
scrobble.log["page_data"] = page_data
scrobble.save(update_fields=["log"])
elif "log" in scrobble_data.keys() and scrobble.log:
scrobble_data["log"] = scrobble.log | scrobble_data["log"]
@ -1436,12 +1434,12 @@ class Scrobble(TimeStampedModel):
if read_log_page:
scrobble_data["log"] = BookLogData(
page_data=[
BookPageLogData(
page_data={
read_log_page: BookPageLogData(
page_number=read_log_page,
start_ts=int(timezone.now().timestamp()),
)
]
}
)
logger.info(
@ -1783,30 +1781,24 @@ class Scrobble(TimeStampedModel):
page_data = self.log.get("page_data")
if page_data:
# --- Sort safely by numeric page_number ---
def safe_page_number(entry):
try:
return int(getattr("page_number", entry), 0)
except (ValueError, TypeError):
return float("inf")
if isinstance(page_data, dict):
logger.warning("Page data is dict, migrate koreader data")
return
valid_pages = sorted(int(k) for k in page_data.keys())
if valid_pages:
self.log["page_start"] = min(valid_pages)
self.log["page_end"] = max(valid_pages)
self.log["pages_read"] = len(set(valid_pages))
elif isinstance(page_data, list):
valid_pages = []
for page in page_data:
try:
valid_pages.append(int(page["page_number"]))
except (ValueError, TypeError):
continue
page_data.sort(key=safe_page_number)
valid_pages = []
for page in page_data:
try:
valid_pages.append(int(page["page_number"]))
except (ValueError, TypeError):
continue
if valid_pages:
self.log["page_start"] = min(valid_pages)
self.log["page_end"] = max(valid_pages)
self.log["pages_read"] = len(set(valid_pages))
if valid_pages:
self.log["page_start"] = min(valid_pages)
self.log["page_end"] = max(valid_pages)
self.log["pages_read"] = len(set(valid_pages))
else:
page_start = self.log.get("page_start")
page_end = self.log.get("page_end")