From b5bfad73efd8cb2cf7ab7ec612798f58c04feacb Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Mon, 20 Oct 2025 15:40:40 -0400 Subject: [PATCH] [books] Allow comic scrobbling to update per page --- poetry.lock | 16 ++++- pyproject.toml | 1 + vrobbler/apps/books/constants.py | 2 + ...comicvine_id_book_issue_number_and_more.py | 33 +++++++++ vrobbler/apps/books/models.py | 47 ++++++++++++- .../apps/books/{ => sources}/comicvine.py | 70 ++++++++++++++----- vrobbler/apps/books/utils.py | 18 +++++ vrobbler/apps/scrobbles/models.py | 27 ++++++- vrobbler/apps/scrobbles/scrobblers.py | 30 ++++++-- 9 files changed, 218 insertions(+), 26 deletions(-) create mode 100644 vrobbler/apps/books/migrations/0029_book_comicvine_id_book_issue_number_and_more.py rename vrobbler/apps/books/{ => sources}/comicvine.py (81%) create mode 100644 vrobbler/apps/books/utils.py diff --git a/poetry.lock b/poetry.lock index 932badd..40aae03 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4756,6 +4756,20 @@ webencodings = ">=0.4" doc = ["sphinx", "sphinx_rtd_theme"] test = ["pytest", "ruff"] +[[package]] +name = "titlecase" +version = "2.4.1" +description = "Python Port of John Gruber's titlecase.pl" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "titlecase-2.4.1.tar.gz", hash = "sha256:7d83a277ccbbda11a2944e78a63e5ccaf3d32f828c594312e4862f9a07f635f5"}, +] + +[package.extras] +regex = ["regex (>=2020.4.4)"] + [[package]] name = "tld" version = "0.13" @@ -5525,4 +5539,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.12" -content-hash = "cd3b566597e09aa444f9af30f95f94f922bf3dca71fbd05c887fb10cbc11d7bf" +content-hash = "2e297ef6f8c524840a381ad793946c87b601d81afd569e882fe58120a5f84626" diff --git a/pyproject.toml b/pyproject.toml index c036e90..2081c15 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ orgparse = "^0.4.20250520" tmdbv3api = "^1.9.0" themoviedb = "^1.0.2" feedparser = "^6.0.12" +titlecase = "^2.4.1" [tool.poetry.group.test] optional = true diff --git a/vrobbler/apps/books/constants.py b/vrobbler/apps/books/constants.py index b21b4e6..2de0721 100644 --- a/vrobbler/apps/books/constants.py +++ b/vrobbler/apps/books/constants.py @@ -5,3 +5,5 @@ BOOKS_TITLES_TO_IGNORE = [ "zb2rhkSwygt9vjkAEBj7tP5KVgFqejJqsJ2W3bYsrgiiKK8XL", "zb2rhchGpo7P27mofV9hYjT63d9ZaQnbQ6LSfzmkvsYzvARif", ] + +READCOMICSONLINE_URL = "https://readcomicsonline.ru" diff --git a/vrobbler/apps/books/migrations/0029_book_comicvine_id_book_issue_number_and_more.py b/vrobbler/apps/books/migrations/0029_book_comicvine_id_book_issue_number_and_more.py new file mode 100644 index 0000000..50c735b --- /dev/null +++ b/vrobbler/apps/books/migrations/0029_book_comicvine_id_book_issue_number_and_more.py @@ -0,0 +1,33 @@ +# Generated by Django 4.2.19 on 2025-10-20 18:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('books', '0028_delete_page'), + ] + + operations = [ + migrations.AddField( + model_name='book', + name='comicvine_id', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='issue_number', + field=models.IntegerField(blank=True, max_length=5, null=True), + ), + migrations.AddField( + model_name='book', + name='original_title', + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name='book', + name='volume_number', + field=models.IntegerField(blank=True, max_length=5, null=True), + ), + ] diff --git a/vrobbler/apps/books/models.py b/vrobbler/apps/books/models.py index 3d53ec2..e510b46 100644 --- a/vrobbler/apps/books/models.py +++ b/vrobbler/apps/books/models.py @@ -26,7 +26,7 @@ from scrobbles.mixins import ( from scrobbles.utils import get_scrobbles_for_media from taggit.managers import TaggableManager from thefuzz import fuzz -from vrobbler.apps.books.comicvine import ( +from vrobbler.apps.books.sources.comicvine import ( ComicVineClient, lookup_comic_from_comicvine, ) @@ -135,6 +135,7 @@ class Book(LongPlayScrobblableMixin): ) title = models.CharField(max_length=255) + original_title = models.CharField(max_length=255, **BNULL) authors = models.ManyToManyField(Author, blank=True) koreader_data_by_hash = models.JSONField(**BNULL) isbn_13 = models.CharField(max_length=255, **BNULL) @@ -145,6 +146,11 @@ class Book(LongPlayScrobblableMixin): publish_date = models.DateField(**BNULL) publisher = models.CharField(max_length=255, **BNULL) first_sentence = models.TextField(**BNULL) + # ComicVine + comicvine_id = models.CharField(max_length=255, **BNULL) + issue_number = models.IntegerField(max_length=5, **BNULL) + volume_number = models.IntegerField(max_length=5, **BNULL) + # OpenLibrary openlibrary_id = models.CharField(max_length=255, **BNULL) cover = models.ImageField(upload_to="books/covers/", **BNULL) cover_small = ImageSpecField( @@ -188,6 +194,43 @@ class Book(LongPlayScrobblableMixin): def get_absolute_url(self): return reverse("books:book_detail", kwargs={"slug": self.uuid}) + @classmethod + def get_from_comicvine(cls, title: str, overwrite: bool = False, force_new: bool =False) -> "Book": + book, created = cls.objects.get_or_create(title=title) + if not created and not overwrite and not force_new: + book, created = cls.objects.get_or_create(original_title=title) + logger.info("Found comic by original title, use force_new=True to override") + return book + + book_dict = lookup_comic_from_comicvine(title) + + if created or overwrite: + author_list = [] + author_dicts = book_dict.pop("author_dicts") + if author_dicts: + for author_dict in author_dicts: + if author_dict.get("authorId"): + author, a_created = Author.objects.get_or_create( + semantic_id=author_dict.get("authorId") + ) + author_list.append(author) + if a_created: + author.name = author_dict.get("name") + author.save() + # TODO enrich author? + ... + + for k, v in book_dict.items(): + setattr(book, k, v) + book.save() + + if author_list: + book.authors.add(*author_list) + genres = book_dict.pop("genres", []) + if genres: + book.genre.add(*genres) + return book + @classmethod def find_or_create( cls, title: str, enrich: bool = False, commit: bool = True @@ -215,6 +258,8 @@ class Book(LongPlayScrobblableMixin): return book book_dict = lookup_book_from_google(title) + if not book_dict or book_dict.get("isbn_10"): + book_dict = lookup_comic_from_comicvine(title) author_list = [] authors = book_dict.pop("authors") diff --git a/vrobbler/apps/books/comicvine.py b/vrobbler/apps/books/sources/comicvine.py similarity index 81% rename from vrobbler/apps/books/comicvine.py rename to vrobbler/apps/books/sources/comicvine.py index b5a3d98..909484c 100644 --- a/vrobbler/apps/books/comicvine.py +++ b/vrobbler/apps/books/sources/comicvine.py @@ -3,7 +3,6 @@ ComicVine API Information & Documentation: https://comicvine.gamespot.com/api/ https://comicvine.gamespot.com/api/documentation """ -import json import logging from django.conf import settings @@ -200,34 +199,71 @@ class ComicVineClient(object): def lookup_comic_from_comicvine(title: str) -> dict: + original_title = title + + issue_number = None + volume_nubmer = None + resource_type = "issue" + if "Issue " in title: + resource_type = "issue" + issue_number = title.split("Issue ")[1] + volume_number = None + if "Volume " in title: + resource_type = "volume" + volume_number = title.split("Volume ")[1] + api_key = getattr(settings, "COMICVINE_API_KEY", "") if not api_key: - logger.warn("No ComicVine API key configured, not looking anything up") + logger.warning("No ComicVine API key configured, not looking anything up") return {} client = ComicVineClient( api_key=getattr(settings, "COMICVINE_API_KEY", None) ) - result = [ - r - for r in client.search(title).get("results") - if r.get("resource_type") == "volume" - ][0] - if "volume" not in result.keys(): - logger.warn("No result found on ComicVine", extra={"title": title}) + raw_results = client.search(title).get("results") + results = [ + r + for r in raw_results + if r.get("resource_type") == resource_type + ] + print(results) + if not results: + logger.warning("No comic found on ComicVine") return {} - title = " ".join([result.get("volume").get("name"), result.get("name)")]) + found_result = None + for result in results: + print("checking ", result.get("issue_number"), " to ", str(issue_number)) + if result.get("issue_number") == str(issue_number): + found_result = result + break + if result.get("volume_number") == str(volume_number): + found_result = result + break + + if not found_result: + found_result = results[0] + + logger.info("ComicVine results", extra={"results": results}) + + if not found_result: + logger.warning("No matches found on ComicVine") + return {} + + title = found_result.get("name") + + if found_result.get("volume"): + title = found_result.get("volume").get("name") + data_dict = { "title": title, - "cover_url": result.get("image").get("original_url"), - "comicvine_data": { - "id": result.get("id"), - "site_detail_url": result.get("site_detail_url"), - "description": result.get("description"), - "image": result.get("image").get("original_url"), - }, + "original_title": original_title, + "issue_number": found_result.get("issue_number"), + "volume_number": found_result.get("volume_number"), + "cover_url": found_result.get("image").get("original_url"), + "comicvine_id": found_result.get("id"), + "comicvine_data": found_result, } return data_dict diff --git a/vrobbler/apps/books/utils.py b/vrobbler/apps/books/utils.py new file mode 100644 index 0000000..1199fc0 --- /dev/null +++ b/vrobbler/apps/books/utils.py @@ -0,0 +1,18 @@ +from titlecase import titlecase + +def parse_readcomicsonline_uri(uri: str) -> tuple: + path = uri.split("comic/")[1] + + parts = path.split('/') + title = "" + volume = 1 + page = 1 + if len(parts) == 2: + title = titlecase(parts[0].replace("-", " ")) + volume = parts[1] + if len(parts) == 3: + title = titlecase(parts[0].replace("-", " ")) + volume = parts[1] + page = parts[2] + + return title, volume, page diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index b7e6096..e9f0478 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -12,7 +12,7 @@ import pytz from beers.models import Beer from boardgames.models import BoardGame from books.koreader import process_koreader_sqlite_file -from books.models import Book, Paper +from books.models import Book, Paper, BookPageLogData, BookLogData from bricksets.models import BrickSet from dataclass_wizard.errors import ParseError from django.conf import settings @@ -1131,6 +1131,8 @@ class Scrobble(TimeStampedModel): media_query = models.Q(**{key: media}) scrobble_data[key + "_id"] = media.id skip_in_progress_check = kwargs.get("skip_in_progress_check", False) + read_log_page = kwargs.get("read_log_page", None) + # Find our last scrobble of this media item (track, video, etc) scrobble = ( @@ -1154,7 +1156,7 @@ class Scrobble(TimeStampedModel): ) return scrobble - if not skip_in_progress_check: + if not skip_in_progress_check or read_log_page: logger.info( f"[create_or_update] check for existing scrobble to update ", extra={ @@ -1170,15 +1172,34 @@ class Scrobble(TimeStampedModel): # If it's marked as stopped, send it through our update mechanism, which will complete it if scrobble and ( scrobble.can_be_updated + or read_log_page or scrobble_data["playback_status"] == "stopped" ): - if "log" in scrobble_data.keys() and scrobble.log: + if read_log_page: + page_list = scrobble.log.get("page_data", []) + if page_list: + for page in page_list: + if not page.get("end_ts", None): + page["end_ts"] = int(timezone.now().timestamp()) + + page_list.append( + BookPageLogData( + page_number=read_log_page, + start_ts=int(timezone.now().timestamp()) + ) + ) + scrobble.log["page_data"] = page_list + scrobble.save(update_fields=["log"]) + elif "log" in scrobble_data.keys() and scrobble.log: scrobble_data["log"] = scrobble.log | scrobble_data["log"] return scrobble.update(scrobble_data) # Discard status before creating scrobble_data.pop("playback_status") + if read_log_page: + scrobble_data["log"] = BookLogData(page_data=BookPageLogData(page_number=read_log_page, start_ts=int(timezone.now().timestamp()))) + logger.info( f"[scrobbling] creating new scrobble", extra={ diff --git a/vrobbler/apps/scrobbles/scrobblers.py b/vrobbler/apps/scrobbles/scrobblers.py index 9f03d93..7217fac 100644 --- a/vrobbler/apps/scrobbles/scrobblers.py +++ b/vrobbler/apps/scrobbles/scrobblers.py @@ -7,7 +7,9 @@ import pendulum import pytz from beers.models import Beer from boardgames.models import BoardGame, BoardGameDesigner, BoardGameLocation -from books.models import Book +from books.constants import READCOMICSONLINE_URL +from books.models import Book, BookLogData, BookPageLogData +from books.utils import parse_readcomicsonline_uri from bricksets.models import BrickSet from dateutil.parser import parse from django.utils import timezone @@ -255,13 +257,31 @@ def manual_scrobble_video_game( def manual_scrobble_book( title: str, user_id: int, action: Optional[str] = None ): - book = Book.find_or_create(title) + log = {} + source = "Vrobbler" + page = None + + if READCOMICSONLINE_URL in title: + title, volume, page = parse_readcomicsonline_uri(title) + title = f"{title} - Issue {volume}" + + if not page: + page = 1 + + log = BookLogData(page_data=BookPageLogData(page_number=page, start_ts=int(timezone.now().timestamp()))) + logger.info("[scrobblers] Book page included in scrobble, should update!") + + source = READCOMICSONLINE_URL + + # TODO: Check for scrobble of this book already and if so, update the page count + + book = Book.find_or_create(title, enrich=True) scrobble_dict = { "user_id": user_id, "timestamp": timezone.now(), "playback_position_seconds": 0, - "source": "Vrobbler", + "source": source, "long_play_complete": False, } @@ -275,7 +295,7 @@ def manual_scrobble_book( }, ) - return Scrobble.create_or_update(book, user_id, scrobble_dict) + return Scrobble.create_or_update(book, user_id, scrobble_dict, read_log_page=page) def manual_scrobble_board_game( @@ -532,6 +552,8 @@ def manual_scrobble_from_url( if content_key == "-i" and "v=" in url: item_id = url.split("v=")[1].split("&")[0] + elif content_key == "-c" and "comics" in url: + item_id = url elif content_key == "-i" and "title/tt" in url: item_id = "tt" + str(item_id)