From a7e8e4f1dc1da73f3081fea27f8a787245de5477 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Sun, 28 May 2023 10:11:24 -0400 Subject: [PATCH] Fix importing from AGB --- vrobbler/apps/videogames/exceptions.py | 2 ++ vrobbler/apps/videogames/retroarch.py | 26 ++++++++++++++++++-------- vrobbler/apps/videogames/scrapers.py | 13 +++++++++---- vrobbler/apps/videogames/utils.py | 23 ++++++++++++++++------- 4 files changed, 45 insertions(+), 19 deletions(-) create mode 100644 vrobbler/apps/videogames/exceptions.py diff --git a/vrobbler/apps/videogames/exceptions.py b/vrobbler/apps/videogames/exceptions.py new file mode 100644 index 0000000..a8aff02 --- /dev/null +++ b/vrobbler/apps/videogames/exceptions.py @@ -0,0 +1,2 @@ +class GameNotFound(Exception): + pass diff --git a/vrobbler/apps/videogames/retroarch.py b/vrobbler/apps/videogames/retroarch.py index bf2e54a..0fb85b3 100644 --- a/vrobbler/apps/videogames/retroarch.py +++ b/vrobbler/apps/videogames/retroarch.py @@ -14,6 +14,7 @@ from videogames.models import VideoGame from videogames.scrapers import scrape_game_name_from_adb from videogames.utils import get_or_create_videogame from vrobbler.apps.scrobbles.exceptions import UserNotFound +from vrobbler.apps.videogames.exceptions import GameNotFound logger = logging.getLogger(__name__) @@ -44,9 +45,7 @@ def load_game_data(directory_path: str, user_tz=None) -> dict: for file in os.listdir(directory): filename = os.fsdecode(file) if not filename.endswith("lrtl"): - logger.info( - f"Found non-gamelog file extension, skipping {filename}" - ) + logger.info(f'Skipping "{filename}", not lrtl file') continue game_name = filename.split(".lrtl")[0].split(" (")[0] @@ -94,14 +93,26 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: for game_name, game_data in game_logs.items(): # Use the retroarch name, because we can't change those but may want to # tweak the found game - mame_name = scrape_game_name_from_adb(game_name) + logger.info(f"Received name {game_name}") + try: + mame_name = scrape_game_name_from_adb(game_name) + except GameNotFound as e: + logger.warning(e) + continue + if mame_name: + logger.info(f"Found name {game_name}") game_name = mame_name found_game = VideoGame.objects.filter(retroarch_name=game_name).first() if not found_game: - found_game = get_or_create_videogame(game_name) + try: + found_game = get_or_create_videogame(game_name) + except GameNotFound as e: + logger.warning(f"Game not found for: {e}") + continue + if found_game: found_game.retroarch_name = game_name found_game.save(update_fields=["retroarch_name"]) @@ -112,9 +123,7 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: stop_timestamp=end_datetime ) if found_scrobble: - logger.info( - f"Found scrobble for {game_name} with stop_timestamp {end_datetime}, not scrobbling" - ) + logger.info(f"Skipping scrobble for game {found_game.id}") continue last_scrobble = found_game.scrobble_set.last() @@ -148,4 +157,5 @@ def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]: ) ) created_scrobbles = Scrobble.objects.bulk_create(new_scrobbles) + logger.info(f"Created {len(created_scrobbles)} scrobbles") return new_scrobbles diff --git a/vrobbler/apps/videogames/scrapers.py b/vrobbler/apps/videogames/scrapers.py index 895d5ea..d957d86 100644 --- a/vrobbler/apps/videogames/scrapers.py +++ b/vrobbler/apps/videogames/scrapers.py @@ -4,6 +4,8 @@ from typing import Optional import requests from bs4 import BeautifulSoup +from vrobbler.apps.videogames.exceptions import GameNotFound + logger = logging.getLogger(__name__) MAME_LOOKUP_URL = "http://adb.arcadeitalia.net/dettaglio_mame.php?game_name={query}&search_id=2" @@ -26,11 +28,14 @@ def scrape_game_name_from_adb(name: str) -> str: title = "" headers = {"User-Agent": "Vrobbler 0.11.12"} url = MAME_LOOKUP_URL.format(query=name) - r = requests.get(url, headers=headers) - if r.status_code == 200: - soup = BeautifulSoup(r.text, "html.parser") - title = _get_title_from_soup(soup).split(" (")[0].split(" / ")[0] + resp = requests.get(url, headers=headers) + if not resp.ok: + raise GameNotFound(f"Lookup failed with code {resp.status_code}") + soup = BeautifulSoup(resp.text, "html.parser") + title = _get_title_from_soup(soup).split(" (")[0].split(" / ")[0] + + logger.info(f"Found title {title}") if title == "Arcade Database": title = "" diff --git a/vrobbler/apps/videogames/utils.py b/vrobbler/apps/videogames/utils.py index 2212669..54ebfec 100644 --- a/vrobbler/apps/videogames/utils.py +++ b/vrobbler/apps/videogames/utils.py @@ -3,10 +3,11 @@ from typing import Optional import requests from django.core.files.base import ContentFile -from videogames.models import VideoGame, VideoGamePlatform - from videogames.howlongtobeat import lookup_game_from_hltb from videogames.igdb import lookup_game_from_igdb +from videogames.models import VideoGame, VideoGamePlatform + +from vrobbler.apps.videogames.exceptions import GameNotFound logger = logging.getLogger(__name__) @@ -45,16 +46,24 @@ def get_or_create_videogame( if "genres" in game_dict.keys(): genres = game_dict.pop("genres") + title = game_dict.get("title") + if not title: + raise GameNotFound(name_or_id) + hltb_id = game_dict.get("hltb_id") igdb_id = game_dict.get("igdb_id") if hltb_id: - game, game_created = VideoGame.objects.get_or_create(hltb_id=hltb_id) - elif igdb_id: - game, game_created = VideoGame.objects.get_or_create(igdb_id=igdb_id) - else: game, game_created = VideoGame.objects.get_or_create( - title=game_dict.get("title") + hltb_id=hltb_id, + title=title, ) + elif igdb_id: + game, game_created = VideoGame.objects.get_or_create( + igdb_id=igdb_id, + title=title, + ) + else: + game, game_created = VideoGame.objects.get_or_create(title=title) if game_created or force_update: VideoGame.objects.filter(pk=game.id).update(**game_dict)