It's not perfect, but at least we get covers now
This commit is contained in:
282
vrobbler/apps/books/management/commands/cleanup_book_metadata.py
Normal file
282
vrobbler/apps/books/management/commands/cleanup_book_metadata.py
Normal file
@ -0,0 +1,282 @@
|
||||
import logging
|
||||
import time
|
||||
|
||||
from django.core.management.base import BaseCommand
|
||||
from django.db import transaction
|
||||
|
||||
from books.constants import READCOMICSONLINE_URL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MISSING_ALL = [
|
||||
"cover",
|
||||
"summary",
|
||||
"isbn",
|
||||
"pages",
|
||||
"language",
|
||||
"publisher",
|
||||
"publish_year",
|
||||
]
|
||||
|
||||
MISSING_GROUPS = {
|
||||
"cover": lambda b: not bool(b.cover),
|
||||
"summary": lambda b: not b.summary,
|
||||
"isbn": lambda b: not b.isbn_13 and not b.isbn_10,
|
||||
"pages": lambda b: b.pages is None,
|
||||
"language": lambda b: not b.language,
|
||||
"publisher": lambda b: not b.publisher,
|
||||
"publish_year": lambda b: b.first_publish_year is None,
|
||||
}
|
||||
|
||||
|
||||
def _book_matches(book, flags):
|
||||
if not flags:
|
||||
return False
|
||||
for flag in flags:
|
||||
fn = MISSING_GROUPS.get(flag)
|
||||
if fn and fn(book):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
help = "Backfill missing metadata on books from Google Books, OpenLibrary, and ComicVine"
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
"--commit",
|
||||
action="store_true",
|
||||
help="Commit changes to the database",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=100,
|
||||
help="Number of books to process per batch (default: 100)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sleep",
|
||||
type=float,
|
||||
default=0.5,
|
||||
help="Seconds to sleep between API calls (default: 0.5)",
|
||||
)
|
||||
for flag in MISSING_ALL:
|
||||
parser.add_argument(
|
||||
f"--missing-{flag}",
|
||||
dest="missing_flags",
|
||||
action="append_const",
|
||||
const=flag,
|
||||
help=f"Process books missing {flag}",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--comics-only",
|
||||
action="store_true",
|
||||
help="Only process books with a readcomicsonline.ru URL",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
dest="all_missing",
|
||||
help="Process books missing any metadata field",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
from books.models import Book
|
||||
|
||||
commit = options["commit"]
|
||||
batch_size = options["batch_size"]
|
||||
sleep_secs = options["sleep"]
|
||||
flags = options.get("missing_flags") or []
|
||||
comics_only = options["comics_only"]
|
||||
all_missing = options["all_missing"]
|
||||
|
||||
if all_missing:
|
||||
flags = MISSING_ALL
|
||||
|
||||
if not flags and not comics_only:
|
||||
self.stdout.write(
|
||||
"No filters specified. Use --all, --missing-*, or --comics-only."
|
||||
)
|
||||
return
|
||||
|
||||
qs = Book.objects.all()
|
||||
if comics_only:
|
||||
qs = qs.filter(readcomics_url__isnull=False)
|
||||
|
||||
if flags:
|
||||
qs = [b for b in qs.iterator() if _book_matches(b, flags)]
|
||||
else:
|
||||
qs = list(qs)
|
||||
|
||||
total = len(qs)
|
||||
self.stdout.write(f"Found {total} books to process")
|
||||
|
||||
if not commit:
|
||||
self.stdout.write(
|
||||
"Dry run — no API calls will be made. Use --commit to run lookups."
|
||||
)
|
||||
return
|
||||
|
||||
enriched = 0
|
||||
skipped = 0
|
||||
stats = {
|
||||
"cover_fixed": 0,
|
||||
"summary_fixed": 0,
|
||||
"isbn_fixed": 0,
|
||||
"pages_fixed": 0,
|
||||
"language_fixed": 0,
|
||||
"publisher_fixed": 0,
|
||||
"publish_year_fixed": 0,
|
||||
}
|
||||
|
||||
for batch_num, offset in enumerate(range(0, len(qs), batch_size)):
|
||||
batch = qs[offset : offset + batch_size]
|
||||
for book in batch:
|
||||
result = self._enrich_book(book, sleep_secs)
|
||||
if result:
|
||||
enriched += 1
|
||||
for key in stats:
|
||||
if result.get(key):
|
||||
stats[key] += 1
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
self.stdout.write(
|
||||
f" Batch {batch_num + 1}: {offset + len(batch)}/{total} — "
|
||||
f"enriched: {enriched}, skipped: {skipped}"
|
||||
)
|
||||
|
||||
self.stdout.write(
|
||||
f"\nResults (commit={commit}):\n"
|
||||
f" Books enriched: {enriched}\n"
|
||||
f" Books skipped: {skipped}\n"
|
||||
f" Covers fixed: {stats['cover_fixed']}\n"
|
||||
f" Summaries fixed:{stats['summary_fixed']}\n"
|
||||
f" ISBNs fixed: {stats['isbn_fixed']}\n"
|
||||
f" Pages fixed: {stats['pages_fixed']}\n"
|
||||
f" Languages fixed:{stats['language_fixed']}\n"
|
||||
f" Publishers fixed:{stats['publisher_fixed']}\n"
|
||||
f" Publish yrs fixed: {stats['publish_year_fixed']}"
|
||||
)
|
||||
|
||||
def _enrich_book(self, book, sleep_secs):
|
||||
from books.sources.comicvine import lookup_comic_from_comicvine
|
||||
from books.sources.google import lookup_book_from_google
|
||||
from books.sources.openlibrary import lookup_book_from_openlibrary as lookup_book_from_ol
|
||||
|
||||
title = book.original_title or book.title
|
||||
author_name = book.author.name if book.author else None
|
||||
book_dict = {}
|
||||
|
||||
is_comic = bool(book.readcomics_url) or (
|
||||
book.issue_number is not None or book.volume_number is not None
|
||||
)
|
||||
if is_comic and READCOMICSONLINE_URL in (book.readcomics_url or ""):
|
||||
cv_data = lookup_comic_from_comicvine(title)
|
||||
if cv_data:
|
||||
book_dict.update(cv_data)
|
||||
|
||||
ol_data = lookup_book_from_ol(title, author=author_name)
|
||||
time.sleep(sleep_secs)
|
||||
google_data = lookup_book_from_google(title)
|
||||
|
||||
if ol_data:
|
||||
for k, v in ol_data.items():
|
||||
book_dict.setdefault(k, v)
|
||||
|
||||
if google_data:
|
||||
for k, v in google_data.items():
|
||||
if v:
|
||||
book_dict.setdefault(k, v)
|
||||
|
||||
if not book_dict:
|
||||
return None
|
||||
|
||||
changed = self._apply(book, book_dict, title)
|
||||
return changed
|
||||
|
||||
def _apply(self, book, data, title):
|
||||
changed = {
|
||||
"cover_fixed": False,
|
||||
"summary_fixed": False,
|
||||
"isbn_fixed": False,
|
||||
"pages_fixed": False,
|
||||
"language_fixed": False,
|
||||
"publisher_fixed": False,
|
||||
"publish_year_fixed": False,
|
||||
}
|
||||
|
||||
update_fields = []
|
||||
|
||||
cover_url = data.pop("cover_url", "")
|
||||
|
||||
if data.get("summary") and not book.summary:
|
||||
book.summary = data["summary"]
|
||||
update_fields.append("summary")
|
||||
changed["summary_fixed"] = True
|
||||
|
||||
if data.get("isbn_13") and not book.isbn_13:
|
||||
book.isbn_13 = data["isbn_13"]
|
||||
update_fields.append("isbn_13")
|
||||
changed["isbn_fixed"] = True
|
||||
|
||||
if data.get("isbn_10") and not book.isbn_10:
|
||||
book.isbn_10 = data["isbn_10"]
|
||||
update_fields.append("isbn_10")
|
||||
changed["isbn_fixed"] = True
|
||||
|
||||
if data.get("pages") and book.pages is None:
|
||||
book.pages = data["pages"]
|
||||
update_fields.append("pages")
|
||||
changed["pages_fixed"] = True
|
||||
|
||||
if data.get("language") and not book.language:
|
||||
book.language = data["language"]
|
||||
update_fields.append("language")
|
||||
changed["language_fixed"] = True
|
||||
|
||||
if data.get("publisher") and not book.publisher:
|
||||
book.publisher = data["publisher"]
|
||||
update_fields.append("publisher")
|
||||
changed["publisher_fixed"] = True
|
||||
|
||||
if data.get("first_publish_year") and book.first_publish_year is None:
|
||||
book.first_publish_year = data["first_publish_year"]
|
||||
update_fields.append("first_publish_year")
|
||||
changed["publish_year_fixed"] = True
|
||||
|
||||
if data.get("openlibrary_id") and not book.openlibrary_id:
|
||||
book.openlibrary_id = data["openlibrary_id"]
|
||||
update_fields.append("openlibrary_id")
|
||||
|
||||
if data.get("comicvine_id") and not book.comicvine_id:
|
||||
book.comicvine_id = data["comicvine_id"]
|
||||
update_fields.append("comicvine_id")
|
||||
|
||||
if data.get("issue_number") and book.issue_number is None:
|
||||
book.issue_number = data["issue_number"]
|
||||
update_fields.append("issue_number")
|
||||
|
||||
if data.get("volume_number") and book.volume_number is None:
|
||||
book.volume_number = data["volume_number"]
|
||||
update_fields.append("volume_number")
|
||||
|
||||
if update_fields:
|
||||
book.save(update_fields=update_fields)
|
||||
self.stdout.write(f" [ENRICHED] {book} — {', '.join(update_fields)}")
|
||||
|
||||
if cover_url and not book.cover:
|
||||
book.save_image_from_url(cover_url)
|
||||
if book.cover:
|
||||
changed["cover_fixed"] = True
|
||||
self.stdout.write(f" [COVER] {book} — cover saved from source")
|
||||
|
||||
genres = data.pop("genres", data.pop("generes", []))
|
||||
if genres:
|
||||
existing = set(book.genre.names())
|
||||
new_genres = [g for g in genres if g not in existing]
|
||||
if new_genres:
|
||||
book.genre.add(*new_genres)
|
||||
self.stdout.write(f" [GENRES] {book} — added {len(new_genres)} genres")
|
||||
|
||||
return changed if any(changed.values()) else None
|
||||
Reference in New Issue
Block a user