diff --git a/PROJECT.org b/PROJECT.org index df0451c..7e7a9bf 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/21] :vrobbler:project:personal: +* Backlog [1/22] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -591,6 +591,10 @@ a helper method to create board game scrobbles given a json blob. It's independent of the email flow it was originally creatdd for ** TODO [#B] Is there way to create unique slugs for media instances :media_types: +** DONE [#A] Video game cleanup script should clear out broken images :metadata:videogames: +:PROPERTIES: +:ID: ca1f1ea9-0f79-082c-5ff7-867671faff4b +:END: * Version 55.1 [1/1] ** DONE [#A] Clean up metadata scrapping for video games :metadata:videogames: :PROPERTIES: diff --git a/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py b/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py index 8b4c1e9..5996552 100644 --- a/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py +++ b/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py @@ -69,6 +69,11 @@ class Command(BaseCommand): type=int, help="Only process a specific game by ID", ) + parser.add_argument( + "--fix-broken-images", + action="store_true", + help="Check and refetch broken/deleted game images (cover, screenshot, hltb_cover)", + ) for flag in MISSING_ALL: parser.add_argument( f"--missing-{flag}", @@ -92,15 +97,16 @@ class Command(BaseCommand): sleep_secs = options["sleep"] force = options["force"] game_id = options["game_id"] + fix_broken_images = options.get("fix_broken_images", False) flags = options.get("missing_flags") or [] all_missing = options["all_missing"] if all_missing: flags = MISSING_ALL - if not flags and not game_id and not force: + if not flags and not game_id and not force and not fix_broken_images: self.stdout.write( - "No filters specified. Use --all, --missing-*, --game-id, or --force." + "No filters specified. Use --all, --missing-*, --game-id, --force, or --fix-broken-images." ) return @@ -135,25 +141,33 @@ class Command(BaseCommand): "release_year_fixed": 0, "igdb_id_found": 0, "hltb_id_found": 0, + "images_fixed": 0, } - for batch_num, offset in enumerate(range(0, len(qs), batch_size)): - batch = qs[offset : offset + batch_size] - for game in batch: - result = self._enrich_game(game, sleep_secs, force) - self._check_retroarch_name(game, title_mismatches) - if result: - enriched += 1 - for key in stats: - if result.get(key): - stats[key] += 1 - else: - skipped += 1 + enriched_any = bool(flags or game_id or force) - self.stdout.write( - f" Batch {batch_num + 1}: {offset + len(batch)}/{total} — " - f"enriched: {enriched}, skipped: {skipped}" - ) + if enriched_any: + for batch_num, offset in enumerate(range(0, len(qs), batch_size)): + batch = qs[offset : offset + batch_size] + for game in batch: + result = self._enrich_game(game, sleep_secs, force) + self._check_retroarch_name(game, title_mismatches) + if result: + enriched += 1 + for key in stats: + if result.get(key): + stats[key] += 1 + else: + skipped += 1 + + self.stdout.write( + f" Batch {batch_num + 1}: {offset + len(batch)}/{total} — " + f"enriched: {enriched}, skipped: {skipped}" + ) + + if fix_broken_images: + broken_stats = self._fix_broken_images(qs, sleep_secs) + stats["images_fixed"] = broken_stats["images_fixed"] self.stdout.write( f"\nResults (commit={commit}):\n" @@ -168,6 +182,8 @@ class Command(BaseCommand): f" IGDB IDs found: {stats['igdb_id_found']}\n" f" HLtB IDs found: {stats['hltb_id_found']}" ) + if fix_broken_images: + self.stdout.write(f" Broken images fixed: {stats['images_fixed']}") if title_mismatches: self.stdout.write("\nTitle vs retroarch_name mismatches (not auto-fixed):") @@ -450,3 +466,64 @@ class Command(BaseCommand): self.stdout.write(f" [TAG] {game} — tagged hltb-enriched") return changed if any(changed.values()) else None + + def _fix_broken_images(self, games, sleep_secs): + from django.core.files.base import ContentFile + + import requests + + from videogames.igdb import lookup_game_from_igdb + from videogames.howlongtobeat import lookup_game_from_hltb + + stats = {"cover_fixed": 0, "screenshot_fixed": 0, "images_fixed": 0} + + for game in games: + for field_name, source in [ + ("cover", "igdb"), + ("screenshot", "igdb"), + ("hltb_cover", "hltb"), + ]: + field = getattr(game, field_name) + if not field.name: + continue + if field.storage.exists(field.name): + continue + + self.stdout.write( + f" [IMAGE] {game} — {field_name} is broken (file missing), refetching…" + ) + + if source == "igdb" and game.igdb_id: + data = lookup_game_from_igdb(str(game.igdb_id)) + time.sleep(sleep_secs) + if not data: + continue + url = data.get("cover_url" if field_name == "cover" else "screenshot_url") + if not url: + continue + r = requests.get(url) + if r.status_code != 200: + continue + fname = f"{game.title}_{game.uuid}.jpg" + getattr(game, field_name).save(fname, ContentFile(r.content), save=True) + stats["images_fixed"] += 1 + self.stdout.write(f" [IMAGE] {game} — {field_name} refetched from IGDB") + + elif source == "hltb" and game.hltb_id: + data = lookup_game_from_hltb(str(game.hltb_id)) + time.sleep(sleep_secs) + if not data: + continue + url = data.get("cover_url") + if not url: + continue + headers = {"User-Agent": "Vrobbler 0.11.12"} + r = requests.get(url, headers=headers) + if r.status_code != 200: + continue + fname = f"{game.title}_cover_{game.uuid}.jpg" + game.hltb_cover.save(fname, ContentFile(r.content), save=True) + stats["images_fixed"] += 1 + self.stdout.write(f" [IMAGE] {game} — hltb_cover refetched from HLtB") + + return stats