From 5f6e75b14e1bf31b34826eb8827f8a9f9b41ac28 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Mon, 15 Jun 2026 12:19:54 -0400 Subject: [PATCH] [books] Fix comic book metadata importing --- PROJECT.org | 31 ++- .../commands/cleanup_book_metadata.py | 33 +++- ...37_book_volume_book_volume_comicvine_id.py | 23 +++ vrobbler/apps/books/models.py | 40 +++- vrobbler/apps/books/sources/comicvine.py | 183 ++++++++++++++++-- 5 files changed, 284 insertions(+), 26 deletions(-) create mode 100644 vrobbler/apps/books/migrations/0037_book_volume_book_volume_comicvine_id.py diff --git a/PROJECT.org b/PROJECT.org index 59adb3d..1a23c2d 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/15] :vrobbler:project:personal: +* Backlog [1/17] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -530,6 +530,35 @@ easily. And our exposure to PII is really low at this point in the project, so we can probably use backtrace=True and diagnose=True to help us root cause bugs faster. +** TODO [#B] Add a /trends/ page that shows trends based on scrobble data :feature:trends:scrobbles: + +*** Description + +This project is a bit invovled. But we should add a top level URL `trends` that shows +various trends as defined either in a static settings file, or dynamically via a database table. + +Examples of trends: + +- How often does the user: + + watch sports while doing a task? + + do a task while watching a video? + * how often do I do + +- trail_scrobble__average_heartrate per trail +- ... + +** DONE [#A] Clean up metadata comicbook enrichment :bug:comics:books:metadata: +:PROPERTIES: +:ID: cd875450-7117-78ca-8be4-9c8b73037dba +:END: + +*** Description + +Still getting wonky results with some comicbooks. Would be nice to be able to +tag a Book as a comicbook, and also gather volume information. I also noticed +that some books that are found in OL never get their comicvine_id populated. We +should make sure we always have comicvine_ids if available. + * Version 51.3 [1/1] ** DONE [#A] Improve speed of index and chart pages :bug:scrobbles:perf: :PROPERTIES: diff --git a/vrobbler/apps/books/management/commands/cleanup_book_metadata.py b/vrobbler/apps/books/management/commands/cleanup_book_metadata.py index e827c83..7ba5959 100644 --- a/vrobbler/apps/books/management/commands/cleanup_book_metadata.py +++ b/vrobbler/apps/books/management/commands/cleanup_book_metadata.py @@ -160,7 +160,10 @@ class Command(BaseCommand): ) def _enrich_book(self, book, sleep_secs): - from books.sources.comicvine import lookup_comic_from_comicvine + from books.sources.comicvine import ( + lookup_comic_from_comicvine, + lookup_issue_by_comicvine_id, + ) from books.sources.google import lookup_book_from_google from books.sources.openlibrary import lookup_book_from_openlibrary as lookup_book_from_ol @@ -168,13 +171,13 @@ class Command(BaseCommand): author_name = book.author.name if book.author else None book_dict = {} - is_comic = bool(book.readcomics_url) or ( - book.issue_number is not None or book.volume_number is not None - ) - if is_comic and READCOMICSONLINE_URL in (book.readcomics_url or ""): + cv_data = None + if book.comicvine_id: + cv_data = lookup_issue_by_comicvine_id(str(book.comicvine_id)) + if not cv_data: cv_data = lookup_comic_from_comicvine(title) - if cv_data: - book_dict.update(cv_data) + if cv_data: + book_dict.update(cv_data) ol_data = lookup_book_from_ol(title, author=author_name) time.sleep(sleep_secs) @@ -261,6 +264,14 @@ class Command(BaseCommand): book.volume_number = data["volume_number"] update_fields.append("volume_number") + if data.get("volume") and not book.volume: + book.volume = data["volume"] + update_fields.append("volume") + + if data.get("volume_comicvine_id") and not book.volume_comicvine_id: + book.volume_comicvine_id = data["volume_comicvine_id"] + update_fields.append("volume_comicvine_id") + if update_fields: book.save(update_fields=update_fields) self.stdout.write(f" [ENRICHED] {book} — {', '.join(update_fields)}") @@ -279,4 +290,12 @@ class Command(BaseCommand): book.genre.add(*new_genres) self.stdout.write(f" [GENRES] {book} — added {len(new_genres)} genres") + tags = data.pop("tags", []) + if tags: + existing_tags = set(book.tags.names()) + new_tags = [t for t in tags if t not in existing_tags] + if new_tags: + book.tags.add(*new_tags) + self.stdout.write(f" [TAGS] {book} — added {', '.join(new_tags)}") + return changed if any(changed.values()) else None diff --git a/vrobbler/apps/books/migrations/0037_book_volume_book_volume_comicvine_id.py b/vrobbler/apps/books/migrations/0037_book_volume_book_volume_comicvine_id.py new file mode 100644 index 0000000..ffc0b54 --- /dev/null +++ b/vrobbler/apps/books/migrations/0037_book_volume_book_volume_comicvine_id.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.29 on 2026-06-15 16:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("books", "0036_alter_book_genre_alter_paper_genre"), + ] + + operations = [ + migrations.AddField( + model_name="book", + name="volume", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="book", + name="volume_comicvine_id", + field=models.CharField(blank=True, max_length=255, null=True), + ), + ] diff --git a/vrobbler/apps/books/models.py b/vrobbler/apps/books/models.py index 4c90daa..20a7f63 100644 --- a/vrobbler/apps/books/models.py +++ b/vrobbler/apps/books/models.py @@ -20,6 +20,7 @@ from books.openlibrary import ( from books.sources.comicvine import ( ComicVineClient, lookup_comic_from_comicvine, + lookup_issue_by_comicvine_id, ) from books.sources.google import lookup_book_from_google from books.sources.openlibrary import ( @@ -150,6 +151,8 @@ class Book(LongPlayScrobblableMixin): first_sentence = models.TextField(**BNULL) # ComicVine comicvine_id = models.CharField(max_length=255, **BNULL) + volume = models.CharField(max_length=255, **BNULL) + volume_comicvine_id = models.CharField(max_length=255, **BNULL) readcomics_url = models.CharField(max_length=255, **BNULL) next_readcomics_url = models.CharField(max_length=255, **BNULL) issue_number = models.IntegerField(**BNULL) @@ -236,13 +239,17 @@ class Book(LongPlayScrobblableMixin): if not book_dict: return book + tags = book_dict.pop("tags", []) + genres = book_dict.pop("genres", book_dict.pop("generes", [])) + for k, v in book_dict.items(): setattr(book, k, v) book.save() - genres = book_dict.get("genres", []) if genres: book.genre.add(*genres) + if tags: + book.tags.add(*tags) return book @classmethod @@ -276,8 +283,10 @@ class Book(LongPlayScrobblableMixin): book_dict = None source_tag = None + tried_comicvine = False if READCOMICSONLINE_URL in url: book_dict = lookup_comic_from_comicvine(title) + tried_comicvine = True if book_dict: source_tag = MediaSourceTag.COMICVINE book_dict["readcomics_url"] = get_comic_issue_url(url) @@ -302,6 +311,16 @@ class Book(LongPlayScrobblableMixin): if ol_data and ol_data.get("cover_url"): book_dict["cover_url"] = ol_data["cover_url"] + # Always try ComicVine as a fallback — it may recognize books that + # OL/Google don't flag as comics + if not tried_comicvine: + cv_data = lookup_comic_from_comicvine(title) + if cv_data: + for k, v in cv_data.items(): + if v: + book_dict.setdefault(k, v) + source_tag = MediaSourceTag.COMICVINE + if not book_dict: logger.warning( "No book found in any source, using data as is", @@ -312,6 +331,7 @@ class Book(LongPlayScrobblableMixin): authors = book_dict.pop("authors", []) cover_url = book_dict.pop("cover_url", "") genres = book_dict.pop("genres", book_dict.pop("generes", [])) + tags = book_dict.pop("tags", []) if authors: for author_str in authors: @@ -331,6 +351,8 @@ class Book(LongPlayScrobblableMixin): book.save_image_from_url(cover_url) if genres: book.genre.add(*genres) + if tags: + book.tags.add(*tags) book.authors.add(*author_list) if source_tag: book.tags.add(source_tag.value) @@ -368,8 +390,14 @@ class Book(LongPlayScrobblableMixin): data = lookup_comic_from_locg(str(self.title)) if not data and COMICVINE_API_KEY: - logger.warn(f"Checking ComicVine for {self.title}") - data = lookup_comic_from_comicvine(str(self.title)) + if self.comicvine_id: + logger.warn( + f"Checking ComicVine by ID for {self.title}" + ) + data = lookup_issue_by_comicvine_id(str(self.comicvine_id)) + if not data: + logger.warn(f"Checking ComicVine for {self.title}") + data = lookup_comic_from_comicvine(str(self.title)) if not data: logger.warn(f"Book not found in any sources: {self.title}") @@ -407,10 +435,10 @@ class Book(LongPlayScrobblableMixin): ) data.pop("pages") - # Pop this, so we can look it up later + # Pop these so they don't get passed to update() cover_url = data.pop("cover_url", "") - subject_key_list = data.pop("subject_key_list", []) + tags = data.pop("tags", []) # Fun trick for updating all fields at once Book.objects.filter(pk=self.id).update(**data) @@ -418,6 +446,8 @@ class Book(LongPlayScrobblableMixin): if subject_key_list: self.genre.add(*subject_key_list) + if tags: + self.tags.add(*tags) if cover_url: r = requests.get(cover_url) diff --git a/vrobbler/apps/books/sources/comicvine.py b/vrobbler/apps/books/sources/comicvine.py index c2cc0dc..ad01b7a 100644 --- a/vrobbler/apps/books/sources/comicvine.py +++ b/vrobbler/apps/books/sources/comicvine.py @@ -17,8 +17,10 @@ class ComicVineClient(object): account on https://comicvine.gamespot.com/ in order to obtain an API key. """ - # All API requests made by this client will be made to this URL. + # All API requests made by this client will be made to these URLs. API_URL = "https://comicvine.gamespot.com/api/search/" + ISSUE_API_URL = "https://comicvine.gamespot.com/api/issue/4000-{issue_id}/" + VOLUME_API_URL = "https://comicvine.gamespot.com/api/volume/4050-{volume_id}/" # A valid User-Agent header must be set in order for our API requests to # be accepted, otherwise our request will be rejected with a @@ -197,6 +199,74 @@ class ComicVineClient(object): raise exception(message) + def get_issue(self, issue_id: str) -> dict: + """ + Fetch a single issue by its ComicVine ID directly from the issue detail + endpoint, which returns richer data than the search endpoint. + + :param issue_id: The ComicVine numeric ID for the issue (e.g. "538480") + :type issue_id: str + + :return: The full JSON response for the issue, or empty dict on failure. + :rtype: dict + """ + params = { + "api_key": self.api_key, + "format": "json", + } + url = self.ISSUE_API_URL.format(issue_id=issue_id) + response = requests.get(url, headers=self.HEADERS, params=params) + + if not response.ok: + self._handle_http_error(response) + + json_data = response.json() + + if json_data.get("status_code") != 1: + error_msg = json_data.get("error", "Unknown ComicVine API error") + logger.error( + "ComicVine API returned status_code %s: %s", + json_data.get("status_code"), + error_msg, + ) + return {} + + return json_data.get("results", {}) + + def get_volume(self, volume_id: str) -> dict: + """ + Fetch a single volume by its ComicVine ID from the volume detail + endpoint. Used to get publisher info and other volume-level metadata. + + :param volume_id: The ComicVine numeric ID for the volume (e.g. "91273") + :type volume_id: str + + :return: The full JSON response for the volume, or empty dict on failure. + :rtype: dict + """ + params = { + "api_key": self.api_key, + "format": "json", + } + url = self.VOLUME_API_URL.format(volume_id=volume_id) + response = requests.get(url, headers=self.HEADERS, params=params) + + if not response.ok: + self._handle_http_error(response) + + json_data = response.json() + + if json_data.get("status_code") != 1: + error_msg = json_data.get("error", "Unknown ComicVine API error") + logger.error( + "ComicVine API returned status_code %s: %s", + json_data.get("status_code"), + error_msg, + ) + return {} + + return json_data.get("results", {}) + def lookup_comic_from_comicvine(title: str) -> dict: original_title = title @@ -238,26 +308,113 @@ def lookup_comic_from_comicvine(title: str) -> dict: if not found_result: found_result = results[0] - title = found_result.get("name") + data_dict = _build_data_dict_from_issue(found_result, original_title) + _enrich_with_volume_data(client, data_dict) + return data_dict - if found_result.get("volume"): - title = found_result.get("volume").get("name") + +def lookup_issue_by_comicvine_id(comicvine_id: str) -> dict: + """ + Look up an issue directly by its ComicVine ID using the issue detail + endpoint. Returns richer data than the search-based lookup. + + :param comicvine_id: The ComicVine numeric ID for the issue (e.g. "538480") + :type comicvine_id: str + + :return: A dict of extracted book metadata, or empty dict on failure. + :rtype: dict + """ + if not comicvine_id: + return {} + + api_key = getattr(settings, "COMICVINE_API_KEY", "") + if not api_key: + logger.warning("No ComicVine API key configured, not looking anything up") + return {} + + client = ComicVineClient(api_key=api_key) + issue_data = client.get_issue(comicvine_id) + if not issue_data: + logger.warning("No issue found on ComicVine for ID %s", comicvine_id) + return {} + + data_dict = _build_data_dict_from_issue(issue_data, issue_data.get("name", "")) + _enrich_with_volume_data(client, data_dict) + return data_dict + + +def _build_data_dict_from_issue(issue_data: dict, original_title: str = "") -> dict: + """ + Build a book metadata dict from a ComicVine issue resource (either from + search results or issue detail endpoint). Both return the same shape of + issue data. + + :param issue_data: The issue resource dict from ComicVine. + :param original_title: The original search term, if any. + + :return: A dict of extracted book metadata. + :rtype: dict + """ + title = issue_data.get("name") + if issue_data.get("volume"): + title = issue_data.get("volume").get("name") cover_url = None - if found_result.get("image"): - cover_url = found_result["image"].get("original_url") + if issue_data.get("image"): + cover_url = issue_data["image"].get("original_url") + + volume_name = None + volume_cv_id = None + publisher_name = None + volume_data = issue_data.get("volume") + if volume_data: + volume_name = volume_data.get("name") + volume_cv_id = volume_data.get("id") + publisher_data = volume_data.get("publisher") + if publisher_data: + publisher_name = publisher_data.get("name") data_dict = { "title": title, "original_title": original_title, - "issue_number": found_result.get("issue_number"), - "volume_number": found_result.get("volume_number"), + "issue_number": issue_data.get("issue_number"), + "volume_number": issue_data.get("volume_number"), + "volume": volume_name, + "volume_comicvine_id": volume_cv_id, + "publisher": publisher_name, "cover_url": cover_url, - "comicvine_id": found_result.get("id"), - "comicvine_data": found_result, - "summary": found_result.get("description"), - "publish_date": found_result.get("cover_date"), - "first_publish_year": (found_result.get("cover_date") or "")[:4], + "comicvine_id": issue_data.get("id"), + "summary": issue_data.get("description"), + "publish_date": issue_data.get("cover_date"), + "first_publish_year": (issue_data.get("cover_date") or "")[:4], + "tags": ["comicbook"], } return data_dict + + +def _enrich_with_volume_data(client: ComicVineClient, data_dict: dict) -> None: + """ + Follow-up a successful issue lookup by fetching the volume detail and + filling in publisher and other volume-level metadata that the issue + endpoint doesn't provide. + + :param client: An initialised ComicVineClient instance. + :param data_dict: The data dict from an issue lookup (mutated in place). + """ + volume_cv_id = data_dict.get("volume_comicvine_id") + if not volume_cv_id: + return + + volume_data = client.get_volume(str(volume_cv_id)) + if not volume_data: + return + + publisher_data = volume_data.get("publisher") + if publisher_data: + publisher_name = publisher_data.get("name") + if publisher_name and not data_dict.get("publisher"): + data_dict["publisher"] = publisher_name + + if not data_dict.get("volume"): + data_dict["volume"] = volume_data.get("name")