From 0061623f7e6dc62d8dedca71366454b4460182a7 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Thu, 18 Jun 2026 15:08:23 -0400 Subject: [PATCH] [videogames] Fix metadata scrapping for video games --- PROJECT.org | 6 +- vrobbler/apps/videogames/howlongtobeat.py | 19 +- vrobbler/apps/videogames/igdb.py | 33 +- .../apps/videogames/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../commands/cleanup_videogame_metadata.py | 452 ++++++++++++++++++ vrobbler/apps/videogames/models.py | 4 + vrobbler/apps/videogames/retroarch.py | 8 + vrobbler/apps/videogames/utils.py | 165 ++++++- 9 files changed, 655 insertions(+), 32 deletions(-) create mode 100644 vrobbler/apps/videogames/management/__init__.py create mode 100644 vrobbler/apps/videogames/management/commands/__init__.py create mode 100644 vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py diff --git a/PROJECT.org b/PROJECT.org index 43e470d..61015e7 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/20] :vrobbler:project:personal: +* Backlog [1/22] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -591,6 +591,10 @@ a helper method to create board game scrobbles given a json blob. It's independent of the email flow it was originally creatdd for ** TODO [#B] Is there way to create unique slugs for media instances :media_types: +** DONE [#A] Clean up metadata scrapping for video games :metadata:videogames: +:PROPERTIES: +:ID: fbc421b5-21a3-4aed-9062-c59192ead065 +:END: * Version 55.0 [3/3] ** DONE [#B] Use pk ID for scrobble detail view, not uuid :scrobbles: :PROPERTIES: diff --git a/vrobbler/apps/videogames/howlongtobeat.py b/vrobbler/apps/videogames/howlongtobeat.py index 863cf44..dd77e60 100644 --- a/vrobbler/apps/videogames/howlongtobeat.py +++ b/vrobbler/apps/videogames/howlongtobeat.py @@ -10,26 +10,27 @@ def hrs_to_secs(hrs: float) -> int: return int(hrs * 60 * 60) -def lookup_game_from_hltb(name_or_id: str) -> Optional[dict]: +def lookup_game_from_hltb(name_or_id: str, search_by_title: bool = False) -> Optional[dict]: """Lookup game on HowLongToBeat.com via HLtB ID or a name string and return the data in a dictonary mapped to our internal game fields """ hltb_game = {} - try: - hltb_id = int(name_or_id) - except ValueError: - hltb_id = None + if not search_by_title: + try: + hltb_id = int(name_or_id) + except ValueError: + hltb_id = None - if hltb_id: - hltb_game = HowLongToBeat().search_from_id(hltb_id) - logger.info(f"Found game on HLtB for ID {hltb_id}") + if hltb_id: + hltb_game = HowLongToBeat().search_from_id(hltb_id) + logger.info(f"Found game on HLtB for ID {hltb_id}") if not hltb_game: results = HowLongToBeat().search(name_or_id) if not results: - logger.warn(f"Lookup of game on HLtB failed for ID {name_or_id}") + logger.warn(f"Lookup of game on HLtB failed via search {name_or_id!r}") return hltb_game = results[0] diff --git a/vrobbler/apps/videogames/igdb.py b/vrobbler/apps/videogames/igdb.py index 0ae46cc..12f4dfe 100644 --- a/vrobbler/apps/videogames/igdb.py +++ b/vrobbler/apps/videogames/igdb.py @@ -19,6 +19,7 @@ GAMES_URL = "https://api.igdb.com/v4/games" ALT_NAMES_URL = "https://api.igdb.com/v4/alternative_names" SCREENSHOT_URL = "https://api.igdb.com/v4/screenshots" COVER_URL = "https://api.igdb.com/v4/covers" +PLATFORMS_URL = "https://api.igdb.com/v4/platforms" IGDB_CLIENT_ID = getattr(settings, "IGDB_CLIENT_ID") IGDB_CLIENT_SECRET = getattr(settings, "IGDB_CLIENT_SECRET") @@ -35,6 +36,20 @@ def get_igdb_token() -> str: return results.get("access_token") +def lookup_platform_names(platform_ids: list, headers: dict) -> list: + """Resolve IGDB platform IDs to platform names""" + if not platform_ids: + return [] + ids_str = ",".join(str(pid) for pid in platform_ids) + body = f"fields name; where id = ({ids_str});" + resp = requests.post(PLATFORMS_URL, data=body, headers=headers) + if resp.status_code != 200: + logger.warn(f"Failed to resolve platform IDs {platform_ids}") + return [] + results = json.loads(resp.content) + return [p["name"] for p in results if "name" in p] + + def lookup_game_id_from_gdb(name: str) -> str: headers = { @@ -62,9 +77,10 @@ def lookup_game_id_from_gdb(name: str) -> str: "details": results.get("details"), }, ) - # Sort our result by IDs so we always get the lowest ID, which is likely to be the least esoteric game - results = sorted(results, key=lambda k: k.get("game", 250000)) - return results[0].get("game", "") + # Sort results by release date (oldest first) to prefer the original game + results = [r for r in results if r.get("game")] + results = sorted(results, key=lambda k: k.get("published_at") or 9999999999) + return results[0].get("game", "") if results else "" def lookup_game_from_igdb(name_or_igdb_id: str) -> Dict: @@ -118,6 +134,16 @@ def lookup_game_from_igdb(name_or_igdb_id: str) -> Dict: for genre in game.get("genres"): genres.append(genre["name"]) + platforms = [] + if "release_dates" in game.keys(): + platform_ids = set() + for rd in game["release_dates"]: + pid = rd.get("platform") + if pid is not None: + platform_ids.add(pid) + if platform_ids: + platforms = lookup_platform_names(list(platform_ids), headers) + game_dict = { "igdb_id": game.get("id"), "title": game.get("name"), @@ -129,6 +155,7 @@ def lookup_game_from_igdb(name_or_igdb_id: str) -> Dict: "release_date": release_date, "summary": game.get("summary"), "genres": genres, + "platforms": platforms, } return game_dict diff --git a/vrobbler/apps/videogames/management/__init__.py b/vrobbler/apps/videogames/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/videogames/management/commands/__init__.py b/vrobbler/apps/videogames/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py b/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py new file mode 100644 index 0000000..8b4c1e9 --- /dev/null +++ b/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py @@ -0,0 +1,452 @@ +import logging +import time + +from django.core.management.base import BaseCommand + +logger = logging.getLogger(__name__) + +MISSING_ALL = [ + "cover", + "screenshot", + "summary", + "rating", + "release_date", + "release_year", + "igdb_id", + "hltb_id", +] + +MISSING_GROUPS = { + "cover": lambda g: not bool(g.cover), + "screenshot": lambda g: not bool(g.screenshot), + "summary": lambda g: not g.summary, + "rating": lambda g: g.rating is None, + "release_date": lambda g: g.release_date is None, + "release_year": lambda g: g.release_year is None, + "igdb_id": lambda g: g.igdb_id is None, + "hltb_id": lambda g: g.hltb_id is None, +} + + +def _game_matches(game, flags): + if not flags: + return False + for flag in flags: + fn = MISSING_GROUPS.get(flag) + if fn and fn(game): + return True + return False + + +class Command(BaseCommand): + help = "Backfill missing metadata on video games from IGDB and HowLongToBeat" + + def add_arguments(self, parser): + parser.add_argument( + "--commit", + action="store_true", + help="Commit changes to the database", + ) + parser.add_argument( + "--batch-size", + type=int, + default=100, + help="Number of games to process per batch (default: 100)", + ) + parser.add_argument( + "--sleep", + type=float, + default=0.5, + help="Seconds to sleep between API calls (default: 0.5)", + ) + parser.add_argument( + "--force", + action="store_true", + help="Re-fetch metadata even if data already exists", + ) + parser.add_argument( + "--game-id", + type=int, + help="Only process a specific game by ID", + ) + for flag in MISSING_ALL: + parser.add_argument( + f"--missing-{flag}", + dest="missing_flags", + action="append_const", + const=flag, + help=f"Process games missing {flag}", + ) + parser.add_argument( + "--all", + action="store_true", + dest="all_missing", + help="Process games missing any metadata field", + ) + + def handle(self, *args, **options): + from videogames.models import VideoGame + + commit = options["commit"] + batch_size = options["batch_size"] + sleep_secs = options["sleep"] + force = options["force"] + game_id = options["game_id"] + flags = options.get("missing_flags") or [] + all_missing = options["all_missing"] + + if all_missing: + flags = MISSING_ALL + + if not flags and not game_id and not force: + self.stdout.write( + "No filters specified. Use --all, --missing-*, --game-id, or --force." + ) + return + + if game_id: + qs = VideoGame.objects.filter(id=game_id) + else: + qs = VideoGame.objects.all() + + if flags: + qs = [g for g in qs.iterator() if _game_matches(g, flags)] + else: + qs = list(qs) + + total = len(qs) + self.stdout.write(f"Found {total} games to process") + + if not commit: + self.stdout.write( + "Dry run — no API calls will be made. Use --commit to run lookups." + ) + return + + title_mismatches = [] + enriched = 0 + skipped = 0 + stats = { + "cover_fixed": 0, + "screenshot_fixed": 0, + "summary_fixed": 0, + "rating_fixed": 0, + "release_date_fixed": 0, + "release_year_fixed": 0, + "igdb_id_found": 0, + "hltb_id_found": 0, + } + + for batch_num, offset in enumerate(range(0, len(qs), batch_size)): + batch = qs[offset : offset + batch_size] + for game in batch: + result = self._enrich_game(game, sleep_secs, force) + self._check_retroarch_name(game, title_mismatches) + if result: + enriched += 1 + for key in stats: + if result.get(key): + stats[key] += 1 + else: + skipped += 1 + + self.stdout.write( + f" Batch {batch_num + 1}: {offset + len(batch)}/{total} — " + f"enriched: {enriched}, skipped: {skipped}" + ) + + self.stdout.write( + f"\nResults (commit={commit}):\n" + f" Games enriched: {enriched}\n" + f" Games skipped: {skipped}\n" + f" Covers fixed: {stats['cover_fixed']}\n" + f" Screenshots fixed: {stats['screenshot_fixed']}\n" + f" Summaries fixed: {stats['summary_fixed']}\n" + f" Ratings fixed: {stats['rating_fixed']}\n" + f" Release dates fixed: {stats['release_date_fixed']}\n" + f" Release years fixed: {stats['release_year_fixed']}\n" + f" IGDB IDs found: {stats['igdb_id_found']}\n" + f" HLtB IDs found: {stats['hltb_id_found']}" + ) + + if title_mismatches: + self.stdout.write("\nTitle vs retroarch_name mismatches (not auto-fixed):") + for retroarch_name, title, game_id in title_mismatches: + self.stdout.write( + f" Game #{game_id}: retroarch_name={retroarch_name!r} vs title={title!r}" + ) + + def _clean_retroarch_name(self, name): + if not name: + return "" + name = name.strip() + if "(" in name: + name = name.split("(")[0].strip() + return name + + def _check_retroarch_name(self, game, mismatches): + if not game.retroarch_name: + return + cleaned = self._clean_retroarch_name(game.retroarch_name) + if cleaned.lower() != game.title.lower(): + mismatches.append((game.retroarch_name, game.title, game.id)) + if "retroarch-mismatch" not in game.tags.names(): + game.tags.add("retroarch-mismatch") + self.stdout.write( + f" [TAG] {game} — tagged as retroarch-mismatch" + ) + + def _enrich_game(self, game, sleep_secs, force): + from videogames.igdb import lookup_game_id_from_gdb, lookup_game_from_igdb + from videogames.howlongtobeat import lookup_game_from_hltb + + search_name = self._clean_retroarch_name(game.retroarch_name) or game.title + + changed = {} + + if not game.hltb_id: + hltb_data = None + if search_name: + hltb_data = lookup_game_from_hltb(search_name, search_by_title=True) + time.sleep(sleep_secs) + if not hltb_data and game.title and game.title != search_name: + hltb_data = lookup_game_from_hltb(game.title, search_by_title=True) + time.sleep(sleep_secs) + if hltb_data: + result = self._apply_hltb_data(game, hltb_data, force) + if result: + changed.update(result) + + igdb_data = None + if not game.igdb_id and search_name: + igdb_id = lookup_game_id_from_gdb(search_name) + time.sleep(sleep_secs) + if igdb_id: + igdb_data = lookup_game_from_igdb(str(igdb_id)) + time.sleep(sleep_secs) + elif game.igdb_id: + igdb_data = lookup_game_from_igdb(str(game.igdb_id)) + time.sleep(sleep_secs) + + if igdb_data: + igdb_title = igdb_data.get("title", "") + igdb_title_clean = self._clean_retroarch_name(igdb_title) + if igdb_title_clean.lower() == search_name.lower(): + if game.igdb_id is None and igdb_data.get("igdb_id"): + game.igdb_id = int(igdb_data["igdb_id"]) + game.save(update_fields=["igdb_id"]) + changed["igdb_id_found"] = True + self.stdout.write(f" [IGDB_ID] {game} — found IGDB ID {game.igdb_id}") + result = self._apply_igdb_data(game, igdb_data, force) + if result: + changed.update(result) + else: + self.stdout.write( + f" [IGDB] {game} — title mismatch (IGDB: {igdb_title!r} vs expected: {search_name!r}), re-searching…" + ) + resolved = False + for candidate in (search_name, game.title if game.title != search_name else None): + if not candidate: + continue + new_id = lookup_game_id_from_gdb(candidate) + time.sleep(sleep_secs) + if not new_id: + continue + new_data = lookup_game_from_igdb(str(new_id)) + time.sleep(sleep_secs) + if not new_data: + continue + new_title = new_data.get("title", "") + new_title_clean = self._clean_retroarch_name(new_title) + if new_title_clean.lower() == candidate.lower(): + game.igdb_id = int(new_id) + if new_title and new_title != game.title: + game.title = new_title + changed["title_updated"] = True + self.stdout.write(f" [TITLE] {game} — updated title to {new_title!r} from IGDB") + game.save(update_fields=["igdb_id"] + (["title"] if changed.get("title_updated") else [])) + changed["igdb_id_found"] = True + self.stdout.write(f" [IGDB_ID] {game} — re-found IGDB ID {game.igdb_id}") + if "igdb-mismatch" in game.tags.names(): + game.tags.remove("igdb-mismatch") + self.stdout.write(f" [TAG] {game} — removed igdb-mismatch tag") + result = self._apply_igdb_data(game, new_data, force) + if result: + changed.update(result) + resolved = True + break + + if not resolved and "igdb-mismatch" not in game.tags.names(): + game.tags.add("igdb-mismatch") + self.stdout.write(f" [TAG] {game} — tagged igdb-mismatch") + + # If retroarch-mismatch tag exists but no longer applies, remove it + if "retroarch-mismatch" in game.tags.names(): + cleaned = self._clean_retroarch_name(game.retroarch_name or "") + if cleaned.lower() == game.title.lower(): + game.tags.remove("retroarch-mismatch") + self.stdout.write(f" [TAG] {game} — removed retroarch-mismatch tag (title now matches)") + + return changed if changed else None + + def _apply_igdb_data(self, game, data, force): + from django.core.files.base import ContentFile + + import requests + + changed = { + "cover_fixed": False, + "screenshot_fixed": False, + "summary_fixed": False, + "rating_fixed": False, + "release_date_fixed": False, + } + + update_fields = [] + + if data.get("alternative_name") and not game.alternative_name: + game.alternative_name = data["alternative_name"] + update_fields.append("alternative_name") + + if data.get("summary") and (not game.summary or force): + game.summary = data["summary"] + update_fields.append("summary") + changed["summary_fixed"] = True + + if data.get("rating") is not None and (game.rating is None or force): + game.rating = data["rating"] + update_fields.append("rating") + changed["rating_fixed"] = True + + if data.get("rating_count") is not None and (game.rating_count is None or force): + game.rating_count = data["rating_count"] + update_fields.append("rating_count") + + if data.get("release_date") and (game.release_date is None or force): + game.release_date = data["release_date"] + update_fields.append("release_date") + changed["release_date_fixed"] = True + + if update_fields: + game.save(update_fields=update_fields) + self.stdout.write(f" [IGDB] {game} — {', '.join(update_fields)}") + + cover_url = data.get("cover_url") + if cover_url: + r = requests.get(cover_url) + if r.status_code == 200: + fname = f"{game.title}_{game.uuid}.jpg" + game.cover.save(fname, ContentFile(r.content), save=True) + changed["cover_fixed"] = True + self.stdout.write(f" [COVER] {game} — cover saved from IGDB") + + screenshot_url = data.get("screenshot_url") + if screenshot_url: + r = requests.get(screenshot_url) + if r.status_code == 200: + fname = f"{game.title}_{game.uuid}.jpg" + game.screenshot.save(fname, ContentFile(r.content), save=True) + changed["screenshot_fixed"] = True + self.stdout.write(f" [SCREENSHOT] {game} — screenshot saved from IGDB") + + genres = data.get("genres", []) + if genres: + existing = set(game.genre.names()) + new_genres = [g for g in genres if g not in existing] + if new_genres: + game.genre.add(*new_genres) + self.stdout.write(f" [GENRES] {game} — added {len(new_genres)} genres") + + platforms = data.get("platforms", []) + if platforms: + existing = set(game.platforms.values_list("name", flat=True)) + new_platforms = [p for p in platforms if p not in existing] + if new_platforms: + from videogames.models import VideoGamePlatform + + for name in new_platforms: + p, _ = VideoGamePlatform.objects.get_or_create(name=name) + game.platforms.add(p) + self.stdout.write(f" [PLATFORMS] {game} — added {len(new_platforms)} platforms") + + if "igdb-enriched" not in game.tags.names(): + game.tags.add("igdb-enriched") + self.stdout.write(f" [TAG] {game} — tagged igdb-enriched") + + return changed if any(changed.values()) else None + + def _apply_hltb_data(self, game, data, force): + from django.core.files.base import ContentFile + + import requests + + changed = { + "hltb_id_found": False, + "release_year_fixed": False, + } + + update_fields = [] + + hltb_title = data.get("title", "") + if hltb_title and hltb_title != game.title: + game.title = hltb_title + update_fields.append("title") + self.stdout.write(f" [TITLE] {game} — updated title to {hltb_title!r}") + + if data.get("hltb_id") and (game.hltb_id is None or force): + game.hltb_id = data["hltb_id"] + update_fields.append("hltb_id") + changed["hltb_id_found"] = True + self.stdout.write(f" [HLTB_ID] {game} — found HLtB ID {data['hltb_id']}") + + if data.get("release_year") and (game.release_year is None or force): + game.release_year = data["release_year"] + update_fields.append("release_year") + changed["release_year_fixed"] = True + + if data.get("main_story_time") and (game.main_story_time is None or force): + game.main_story_time = data["main_story_time"] + update_fields.append("main_story_time") + + if data.get("main_extra_time") and (game.main_extra_time is None or force): + game.main_extra_time = data["main_extra_time"] + update_fields.append("main_extra_time") + + if data.get("completionist_time") and (game.completionist_time is None or force): + game.completionist_time = data["completionist_time"] + update_fields.append("completionist_time") + + if data.get("hltb_score") is not None and (game.hltb_score is None or force): + game.hltb_score = data["hltb_score"] + update_fields.append("hltb_score") + + if update_fields: + game.save(update_fields=update_fields) + self.stdout.write(f" [HLTB] {game} — {', '.join(update_fields)}") + + cover_url = data.get("cover_url") + if cover_url: + headers = {"User-Agent": "Vrobbler 0.11.12"} + r = requests.get(cover_url, headers=headers) + if r.status_code == 200: + fname = f"{game.title}_cover_{game.uuid}.jpg" + game.hltb_cover.save(fname, ContentFile(r.content), save=True) + self.stdout.write(f" [HLTB_COVER] {game} — cover saved from HLtB") + + platforms = data.get("platforms", []) + if platforms: + existing = set(game.platforms.values_list("name", flat=True)) + new_platforms = [p for p in platforms if p not in existing] + if new_platforms: + from videogames.models import VideoGamePlatform + + for name in new_platforms: + p, _ = VideoGamePlatform.objects.get_or_create(name=name) + game.platforms.add(p) + self.stdout.write(f" [PLATFORMS] {game} — added {len(new_platforms)} platforms") + + if "hltb-enriched" not in game.tags.names(): + game.tags.add("hltb-enriched") + self.stdout.write(f" [TAG] {game} — tagged hltb-enriched") + + return changed if any(changed.values()) else None diff --git a/vrobbler/apps/videogames/models.py b/vrobbler/apps/videogames/models.py index 66e1896..ffc7ffc 100644 --- a/vrobbler/apps/videogames/models.py +++ b/vrobbler/apps/videogames/models.py @@ -215,12 +215,16 @@ class VideoGame(LongPlayScrobblableMixin): def fix_metadata(self, force_update: bool = False): from videogames.utils import ( get_or_create_videogame, + load_game_data_from_hltb, load_game_data_from_igdb, ) if self.hltb_id and force_update: get_or_create_videogame(str(self.hltb_id), force_update) + if not self.hltb_id: + load_game_data_from_hltb(self.id) + if not self.igdb_id: # This almost never works without intervention self.igdb_id = lookup_game_id_from_gdb(self.title) diff --git a/vrobbler/apps/videogames/retroarch.py b/vrobbler/apps/videogames/retroarch.py index 18766f7..7f320fc 100644 --- a/vrobbler/apps/videogames/retroarch.py +++ b/vrobbler/apps/videogames/retroarch.py @@ -153,6 +153,13 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: continue logger.info(f"Queued scrobble for game {found_game.id}") + + log_data = {"emulated": True} + if last_scrobble and last_scrobble.log: + prev = last_scrobble.log + if prev.get("emulator"): + log_data["emulator"] = prev["emulator"] + new_scrobbles.append( Scrobble( video_game_id=found_game.id, @@ -168,6 +175,7 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: user_id=user_id, source="Retroarch", media_type=Scrobble.MediaType.VIDEO_GAME, + log=log_data, ) ) created_scrobbles = Scrobble.objects.bulk_create(new_scrobbles) diff --git a/vrobbler/apps/videogames/utils.py b/vrobbler/apps/videogames/utils.py index 8b4e82d..3d4e0e3 100644 --- a/vrobbler/apps/videogames/utils.py +++ b/vrobbler/apps/videogames/utils.py @@ -7,8 +7,6 @@ from videogames.howlongtobeat import lookup_game_from_hltb from videogames.igdb import lookup_game_from_igdb from videogames.models import VideoGame, VideoGamePlatform -from vrobbler.apps.videogames.exceptions import GameNotFound - logger = logging.getLogger(__name__) @@ -16,22 +14,33 @@ def get_or_create_videogame( name_or_id: str, force_update: bool = False, ) -> Optional[VideoGame]: - """Look up game by name or ID from HowLongToBeat""" + """Look up game by name or ID from HowLongToBeat, then enrich with IGDB""" - game_dict = lookup_game_from_hltb(name_or_id) + hltb_data = lookup_game_from_hltb(name_or_id) - if not game_dict: - game_dict = lookup_game_from_igdb(name_or_id) + if hltb_data: + game = _create_update_from_dict(hltb_data, force_update) + else: + igdb_data = lookup_game_from_igdb(name_or_id) + if igdb_data: + game = _create_update_from_dict(igdb_data, force_update) + else: + return None - if not game_dict: - return + if game: + game.fix_metadata() + return game + + +def _create_update_from_dict( + game_dict: dict, force_update: bool = False +) -> Optional[VideoGame]: - # Create missing platforms and prep for loading after create platform_ids = [] if "platforms" in game_dict.keys(): platforms = game_dict.get("platforms", []) if platforms: - for platform in game_dict.get("platforms", []): + for platform in platforms: p, _created = VideoGamePlatform.objects.get_or_create(name=platform) platform_ids.append(p.id) game_dict.pop("platforms") @@ -48,7 +57,7 @@ def get_or_create_videogame( title = game_dict.get("title") if not title: - raise GameNotFound(name_or_id) + return None hltb_id = game_dict.get("hltb_id") igdb_id = game_dict.get("igdb_id") @@ -69,21 +78,19 @@ def get_or_create_videogame( VideoGame.objects.filter(pk=game.id).update(**game_dict) game.refresh_from_db() - # Associate plaforms if platform_ids: game.platforms.add(*platform_ids) if genres: game.genre.add(*genres) - if not game.screenshot and screenshot_url: + if screenshot_url: r = requests.get(screenshot_url) if r.status_code == 200: fname = f"{game.title}_{game.uuid}.jpg" game.screenshot.save(fname, ContentFile(r.content), save=True) - # Go get cover image if the URL is present - if cover_url and not game.hltb_cover: + if cover_url: headers = {"User-Agent": "Vrobbler 0.11.12"} r = requests.get(cover_url, headers=headers) logger.debug(r.status_code) @@ -91,12 +98,89 @@ def get_or_create_videogame( fname = f"{game.title}_cover_{game.uuid}.jpg" game.hltb_cover.save(fname, ContentFile(r.content), save=True) logger.debug("Loaded cover image from HLtB") - game.fix_metadata() + + tag = "hltb-enriched" if hltb_id else "igdb-enriched" + if tag not in game.tags.names(): + game.tags.add(tag) + logger.info(f"Game {game} tagged {tag}") return game -def load_game_data_from_igdb(game_id: int, igdb_id: str = "") -> Optional[VideoGame]: +def load_game_data_from_hltb( + game_id: int, expected_title: str = "" +) -> Optional[VideoGame]: + """Look up HLtB data for an existing game and apply it""" + game = VideoGame.objects.filter(id=game_id).first() + if not game: + logger.warn(f"Video game with ID {game_id} not found") + return + + logger.info(f"Looking up HLtB data for {game}") + hltb_data = lookup_game_from_hltb(game.title, search_by_title=True) + if not hltb_data: + logger.warn(f"No HLtB data found for {game}") + return + + update_fields = [] + + hltb_title = hltb_data.get("title", "") + if hltb_title and hltb_title != game.title: + game.title = hltb_title + update_fields.append("title") + logger.info(f"Game {game.id} title updated to {hltb_title!r}") + + if hltb_data.get("hltb_id") and (game.hltb_id is None): + game.hltb_id = hltb_data["hltb_id"] + update_fields.append("hltb_id") + + if hltb_data.get("release_year") and (game.release_year is None): + game.release_year = hltb_data["release_year"] + update_fields.append("release_year") + + if hltb_data.get("main_story_time") and (game.main_story_time is None): + game.main_story_time = hltb_data["main_story_time"] + update_fields.append("main_story_time") + + if hltb_data.get("main_extra_time") and (game.main_extra_time is None): + game.main_extra_time = hltb_data["main_extra_time"] + update_fields.append("main_extra_time") + + if hltb_data.get("completionist_time") and (game.completionist_time is None): + game.completionist_time = hltb_data["completionist_time"] + update_fields.append("completionist_time") + + if hltb_data.get("hltb_score") is not None and (game.hltb_score is None): + game.hltb_score = hltb_data["hltb_score"] + update_fields.append("hltb_score") + + if update_fields: + game.save(update_fields=update_fields) + + platforms = hltb_data.get("platforms", []) + if platforms: + for name in platforms: + p, _ = VideoGamePlatform.objects.get_or_create(name=name) + game.platforms.add(p) + + cover_url = hltb_data.get("cover_url") + if cover_url: + headers = {"User-Agent": "Vrobbler 0.11.12"} + r = requests.get(cover_url, headers=headers) + if r.status_code == 200: + fname = f"{game.title}_cover_{game.uuid}.jpg" + game.hltb_cover.save(fname, ContentFile(r.content), save=True) + + if "hltb-enriched" not in game.tags.names(): + game.tags.add("hltb-enriched") + logger.info(f"Game {game} tagged hltb-enriched") + + return game + + +def load_game_data_from_igdb( + game_id: int, igdb_id: str = "", expected_title: str = "" +) -> Optional[VideoGame]: """Look up game, if it doesn't exist, lookup data from igdb""" game = VideoGame.objects.filter(id=game_id).first() if not game: @@ -116,25 +200,68 @@ def load_game_data_from_igdb(game_id: int, igdb_id: str = "") -> Optional[VideoG logger.warn(f"No game data found on IGDB for ID {igdb_id}") return + igdb_title = game_dict.get("title", "") + igdb_title_clean = igdb_title.split(" (")[0].strip() if " (" in igdb_title else igdb_title + expected = expected_title or game.title + if igdb_title_clean.lower() != expected.lower(): + logger.info( + f"IGDB title {igdb_title!r} doesn't match expected {expected!r} for {game} — re-searching…" + ) + from videogames.igdb import lookup_game_id_from_gdb + + new_id = lookup_game_id_from_gdb(expected) + if new_id: + new_data = lookup_game_from_igdb(str(new_id)) + if new_data: + new_title = new_data.get("title", "") + new_title_clean = new_title.split(" (")[0].strip() if " (" in new_title else new_title + if new_title_clean.lower() == expected.lower(): + game_dict = new_data + igdb_id = int(new_id) + if game.igdb_id != igdb_id: + game.igdb_id = igdb_id + game.save(update_fields=["igdb_id"]) + logger.info(f"Game {game} IGDB ID updated to {igdb_id}") + + igdb_title = game_dict.get("title", "") + igdb_title_clean = igdb_title.split(" (")[0].strip() if " (" in igdb_title else igdb_title + if igdb_title_clean.lower() != expected.lower(): + if "igdb-mismatch" not in game.tags.names(): + game.tags.add("igdb-mismatch") + logger.info( + f"Game {game} tagged igdb-mismatch (IGDB: {igdb_title!r} vs expected: {expected!r})" + ) + return + screenshot_url = game_dict.pop("screenshot_url") cover_url = game_dict.pop("cover_url") genres = game_dict.pop("genres") + platforms = game_dict.pop("platforms", []) VideoGame.objects.filter(pk=game.id).update(**game_dict) game.refresh_from_db() game.genre.add(*genres) - if not game.screenshot and screenshot_url: + if platforms: + for name in platforms: + p, _ = VideoGamePlatform.objects.get_or_create(name=name) + game.platforms.add(p) + + if screenshot_url: r = requests.get(screenshot_url) if r.status_code == 200: fname = f"{game.title}_{game.uuid}.jpg" game.screenshot.save(fname, ContentFile(r.content), save=True) - if not game.cover and cover_url: + if cover_url: r = requests.get(cover_url) if r.status_code == 200: fname = f"{game.title}_{game.uuid}.jpg" game.cover.save(fname, ContentFile(r.content), save=True) + if "igdb-enriched" not in game.tags.names(): + game.tags.add("igdb-enriched") + logger.info(f"Game {game} tagged igdb-enriched") + return game