diff --git a/PROJECT.org b/PROJECT.org index c8510b4..8a02e8f 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [1/26] :vrobbler:project:personal: +* Backlog [2/26] :vrobbler:project:personal: ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: @@ -617,7 +617,10 @@ boardgame_variant_ids assocaited with their log data, and a variant is created for one specific board game. -** TODO [#A] Lookup all Expansions for a game when creating it :boardgames: +** DONE [#A] Lookup all Expansions for a game when creating it :boardgames: +:PROPERTIES: +:ID: 8a84b06d-555c-4701-9058-ff364c89c198 +:END: *** Description diff --git a/vrobbler/apps/boardgames/management/commands/fetch_expansions.py b/vrobbler/apps/boardgames/management/commands/fetch_expansions.py new file mode 100644 index 0000000..e1e41c6 --- /dev/null +++ b/vrobbler/apps/boardgames/management/commands/fetch_expansions.py @@ -0,0 +1,78 @@ +import logging + +from django.core.management.base import BaseCommand + +from boardgames.models import BoardGame +from boardgames.sources.bgg import lookup_boardgame_from_bgg +from boardgames.utils import fetch_and_link_expansions + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Fetch and link expansions for existing board games from BGG" + + def add_arguments(self, parser): + parser.add_argument( + "--commit", + action="store_true", + help="Persist changes to the database", + ) + parser.add_argument( + "--bggeek-id", + type=str, + help="Only process a single game by BGG ID", + ) + + def handle(self, *args, **options): + commit = options.get("commit", False) + bggeek_id = options.get("bggeek_id") + + games = BoardGame.objects.exclude(bggeek_id__isnull=True).exclude( + bggeek_id="" + ) + if bggeek_id: + games = games.filter(bggeek_id=bggeek_id) + + total = games.count() + self.stdout.write(f"Found {total} board games with BGG IDs") + + if total == 0: + return + + updated = 0 + for game in games.iterator(chunk_size=100): + try: + data = lookup_boardgame_from_bgg(lookup_id=str(game.bggeek_id)) + except Exception as e: + self.stdout.write( + self.style.WARNING( + f"Failed to fetch BGG data for {game.title} ({game.bggeek_id}): {e}" + ) + ) + continue + + expansions = data.get("expansions", []) + if not expansions: + continue + + if commit: + fetch_and_link_expansions(game, expansions) + updated += 1 + self.stdout.write( + f" Linked {len(expansions)} expansions to {game.title}" + ) + else: + self.stdout.write( + f" Would link {len(expansions)} expansions to {game.title}" + ) + updated += 1 + + if commit: + self.stdout.write( + self.style.SUCCESS(f"Updated {updated} games (changes committed)") + ) + else: + self.stdout.write( + f"Would update {updated} games (pass --commit to persist)" + ) diff --git a/vrobbler/apps/boardgames/models.py b/vrobbler/apps/boardgames/models.py index 0e9a7da..0d0e6a2 100644 --- a/vrobbler/apps/boardgames/models.py +++ b/vrobbler/apps/boardgames/models.py @@ -71,7 +71,7 @@ class BoardGameLogData(BaseLogData, LongPlayLogData): difficulty: Optional[int] = None solo: Optional[bool] = None two_handed: Optional[bool] = None - expansion_ids: Optional[int] = None + expansion_ids: Optional[list[int]] = None moves: Optional[list] = None rated: Optional[str] = None speed: Optional[str] = None @@ -112,6 +112,13 @@ class BoardGameLogData(BaseLogData, LongPlayLogData): required=False, widget=forms.SelectMultiple(attrs={"size": 5}), ), + "expansion_ids": forms.ModelMultipleChoiceField( + queryset=BoardGame.objects.filter( + expansion_for_boardgame__isnull=False + ), + required=False, + widget=forms.SelectMultiple(attrs={"size": 5}), + ), } fields.update(custom_fields) return fields @@ -326,18 +333,28 @@ class BoardGame(ScrobblableMixin): if not data: data = lookup_boardgame_from_bgg(str(self.bggeek_id)) - cover_url = data.pop("cover_url") - year = data.pop("year_published") - publisher_name = data.pop("publisher_name") + expansions = data.pop("expansions", []) + cover_url = data.pop("cover_url", "") + year_published = data.pop("year_published", None) + if year_published is None: + year_published = data.pop("published_year", None) + publisher_name = data.pop("publisher_name", "") - if year: - data["published_year"] = int(year) + if year_published: + data["published_year"] = int(year_published) if not data["min_players"]: data.pop("min_players") if not data["min_players"]: data.pop("max_players") + # Pop extra BGG metadata that isn't a model field + data.pop("mechanics", None) + data.pop("categories", None) + data.pop("designers", None) + data.pop("publishers", None) + data.pop("publisher", None) + # Fun trick for updating all fields at once BoardGame.objects.filter(pk=self.id).update(**data) self.refresh_from_db() @@ -353,6 +370,10 @@ class BoardGame(ScrobblableMixin): if cover_url and not self.cover: self.save_image_from_url(cover_url) + from boardgames.utils import fetch_and_link_expansions + + fetch_and_link_expansions(self, expansions) + def save_image_from_url(self, url): headers = {"User-Agent": "Vrobbler 0.11.12"} r = requests.get(url, headers=headers) @@ -372,21 +393,7 @@ class BoardGame(ScrobblableMixin): "Board game exists in database.", extra={"lookup_id": lookup_id, "data": data}, ) - return game - - -class BoardGameVariant(TimeStampedModel): - name = models.CharField(max_length=255) - board_game = models.ForeignKey( - BoardGame, - on_delete=models.CASCADE, - related_name="variants", - ) - description = models.TextField(**BNULL) - uuid = models.UUIDField(default=uuid4, editable=False, **BNULL) - - def __str__(self) -> str: - return f"{self.name} ({self.board_game.title})" + return game if data.get("bggId"): bgg_data = lookup_boardgame_from_bgg(lookup_id=data.get("bggId")) @@ -398,6 +405,7 @@ class BoardGameVariant(TimeStampedModel): else: bgg_data = lookup_boardgame_from_bgg(title=lookup_id) + expansions = bgg_data.pop("expansions", []) mechanics = bgg_data.pop("mechanics", []) designers = bgg_data.pop("designers", []) categories = bgg_data.pop("categories", []) @@ -430,4 +438,22 @@ class BoardGameVariant(TimeStampedModel): publisher, _ = BoardGamePublisher.objects.get_or_create(name=name) game.publishers.add(publisher) + from boardgames.utils import fetch_and_link_expansions + + fetch_and_link_expansions(game, expansions) + return game + + +class BoardGameVariant(TimeStampedModel): + name = models.CharField(max_length=255) + board_game = models.ForeignKey( + BoardGame, + on_delete=models.CASCADE, + related_name="variants", + ) + description = models.TextField(**BNULL) + uuid = models.UUIDField(default=uuid4, editable=False, **BNULL) + + def __str__(self) -> str: + return f"{self.name} ({self.board_game.title})" diff --git a/vrobbler/apps/boardgames/sources/bgg.py b/vrobbler/apps/boardgames/sources/bgg.py index 7330863..68d906b 100644 --- a/vrobbler/apps/boardgames/sources/bgg.py +++ b/vrobbler/apps/boardgames/sources/bgg.py @@ -36,5 +36,8 @@ def lookup_boardgame_from_bgg( game_dict["publishers"] = game.publishers if game.publishers: game_dict["publisher"] = game.publishers[0] + game_dict["expansions"] = [ + {"id": exp.id, "name": exp.name} for exp in game.expansions + ] return game_dict diff --git a/vrobbler/apps/boardgames/tests/test_models.py b/vrobbler/apps/boardgames/tests/test_models.py index c4a2134..7dbb655 100644 --- a/vrobbler/apps/boardgames/tests/test_models.py +++ b/vrobbler/apps/boardgames/tests/test_models.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + import pytest from django.contrib.auth import get_user_model @@ -63,3 +65,161 @@ def test_board_game_variant_cascade_delete(): ) game.delete() assert BoardGameVariant.objects.count() == 0 + + +def _mock_bgg_game(bggeek_id, title, expansions=None): + """Build a fake BGG game object shape used by lookup_boardgame_from_bgg.""" + class FakeGame: + id = bggeek_id + name = title + description = f"Description of {title}" + yearpublished = 2020 + image = "https://example.com/cover.jpg" + minplayers = 1 + maxplayers = 4 + minage = 8 + rating_average = 7.5 + bgg_rank = 100 + playingtime = 60 + mechanics = [] + categories = [] + designers = [] + publishers = [] + + @property + def expansions(self): + if expansions is None: + return [] + return expansions + + return FakeGame() + + +def _mock_bgg_client(return_game): + """Return a callable that creates a fake BGGClient instance.""" + class FakeBGGClient: + def __init__(self, access_token=None): + pass + + def game(self, game_id=None, name=None): + return return_game + + return FakeBGGClient() + + +@pytest.mark.django_db +@patch("boardgames.models.requests.get") +@patch("boardgames.sources.bgg.BGGClient") +def test_find_or_create_links_expansions(mock_bgg, mock_get): + exp1 = type("Thing", (), {"id": 200, "name": "Expansion 1"})() + exp2 = type("Thing", (), {"id": 201, "name": "Expansion 2"})() + base_game = _mock_bgg_game("100", "Base Game", expansions=[exp1, exp2]) + exp1_game = _mock_bgg_game("200", "Expansion 1") + exp2_game = _mock_bgg_game("201", "Expansion 2") + mock_bgg.side_effect = [ + _mock_bgg_client(base_game), + _mock_bgg_client(exp1_game), + _mock_bgg_client(exp2_game), + ] + mock_get.return_value.status_code = 200 + mock_get.return_value.content = b"fake_image_data" + + game = BoardGame.find_or_create("100") + assert game.title == "Base Game" + + expansions = BoardGame.objects.filter(expansion_for_boardgame=game) + assert expansions.count() == 2 + assert {e.title for e in expansions} == {"Expansion 1", "Expansion 2"} + + +@pytest.mark.django_db +@patch("boardgames.sources.bgg.BGGClient") +def test_find_or_create_skips_expansions_for_existing_game(mock_bgg): + game = BoardGame.objects.create(title="Base Game", bggeek_id="100") + mock_bgg.assert_not_called() + + result = BoardGame.find_or_create("100") + assert result == game + + +@pytest.mark.django_db +@patch("boardgames.models.requests.get") +@patch("boardgames.sources.bgg.BGGClient") +def test_fetch_and_link_expansions(mock_bgg, mock_get): + game = BoardGame.objects.create(title="Base Game", bggeek_id="100") + exp_game = _mock_bgg_game("200", "Expansion 1") + mock_bgg.side_effect = [_mock_bgg_client(exp_game)] + mock_get.return_value.status_code = 200 + mock_get.return_value.content = b"fake_image_data" + + from boardgames.utils import fetch_and_link_expansions + + fetch_and_link_expansions(game, [{"id": 200, "name": "Expansion 1"}]) + + expansion = BoardGame.objects.get(bggeek_id="200") + assert expansion.expansion_for_boardgame == game + + +@pytest.mark.django_db +@patch("boardgames.models.requests.get") +@patch("boardgames.sources.bgg.BGGClient") +def test_fix_metadata_links_expansions(mock_bgg, mock_get): + game = BoardGame.objects.create(title="Base Game", bggeek_id="100") + exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})() + base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing]) + exp_game = _mock_bgg_game("200", "Expansion 1") + # first call = fix_metadata -> lookup_boardgame_from_bgg(lookup_id='100') + # second call = find_or_create inside fetch_and_link_expansions for expansion + mock_bgg.side_effect = [ + _mock_bgg_client(base_with_exp), + _mock_bgg_client(exp_game), + ] + mock_get.return_value.status_code = 200 + mock_get.return_value.content = b"fake_image_data" + + game.fix_metadata(force_update=True) + + expansion = BoardGame.objects.get(bggeek_id="200") + assert expansion.expansion_for_boardgame == game + + +@pytest.mark.django_db +@patch("boardgames.sources.bgg.BGGClient") +def test_management_command_fetch_expansions_dry_run(mock_bgg, capsys): + from django.core.management import call_command + + game = BoardGame.objects.create(title="Base Game", bggeek_id="100") + exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})() + base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing]) + mock_bgg.side_effect = [_mock_bgg_client(base_with_exp)] + + call_command("fetch_expansions") + captured = capsys.readouterr() + assert "Would link 1 expansions" in captured.out + + assert BoardGame.objects.filter(bggeek_id="200").count() == 0 + + +@pytest.mark.django_db +@patch("boardgames.models.requests.get") +@patch("boardgames.sources.bgg.BGGClient") +def test_management_command_fetch_expansions_commit(mock_bgg, mock_get, capsys): + from django.core.management import call_command + + game = BoardGame.objects.create(title="Base Game", bggeek_id="100") + exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})() + exp_game = _mock_bgg_game("200", "Expansion 1") + base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing]) + mock_bgg.side_effect = [ + _mock_bgg_client(base_with_exp), + _mock_bgg_client(exp_game), + ] + mock_get.return_value.status_code = 200 + mock_get.return_value.content = b"fake_image_data" + + call_command("fetch_expansions", commit=True) + captured = capsys.readouterr() + assert "Updated 1 games" in captured.out + + expansion = BoardGame.objects.get(bggeek_id="200") + assert expansion.expansion_for_boardgame == game diff --git a/vrobbler/apps/boardgames/utils.py b/vrobbler/apps/boardgames/utils.py index b5ad499..92e86a0 100644 --- a/vrobbler/apps/boardgames/utils.py +++ b/vrobbler/apps/boardgames/utils.py @@ -1,4 +1,5 @@ import logging +from typing import Any from boardgames.models import BoardGame, BoardGameVariant @@ -35,3 +36,27 @@ def board_names_to_variants( ) variant_ids.append(variant.id) return variant_ids + + +def fetch_and_link_expansions( + board_game: BoardGame, expansions_data: list[dict[str, Any]] +) -> None: + """Given a board game and a list of expansion dicts (with 'id' and 'name'), + find or create each expansion BoardGame and link it via expansion_for_boardgame. + """ + for exp_data in expansions_data: + exp_id = exp_data.get("id") + if not exp_id: + continue + expansion = BoardGame.find_or_create(str(exp_id)) + if expansion and expansion.id != board_game.id: + expansion.expansion_for_boardgame = board_game + expansion.save(update_fields=["expansion_for_boardgame"]) + logger.info( + "Linked expansion to board game", + extra={ + "board_game_id": board_game.id, + "expansion_id": expansion.id, + "expansion_name": expansion.title, + }, + ) diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index 7fce833..71e6c65 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -1235,6 +1235,15 @@ class ScrobbleDetailView(DetailView): def get_form_class(self): return self.object.media_obj.logdata_cls().form() + def _update_expansion_ids_queryset(self, form): + from boardgames.models import BoardGame + + if isinstance(self.object.media_obj, BoardGame) and "expansion_ids" in form.fields: + expansions = BoardGame.objects.filter( + expansion_for_boardgame=self.object.media_obj + ) + form.fields["expansion_ids"].queryset = expansions + def get_form(self): FormClass = self.get_form_class() @@ -1245,12 +1254,15 @@ class ScrobbleDetailView(DetailView): else: log["notes"] = self.object.logdata.notes_as_str(separator="\n") - return FormClass(initial=log) + form = FormClass(initial=log) + self._update_expansion_ids_queryset(form) + return form def post(self, request, *args, **kwargs): self.object = self.get_object() FormClass = self.get_form_class() form = FormClass(request.POST) + self._update_expansion_ids_queryset(form) if form.is_valid(): data = form.cleaned_data.copy() @@ -1263,6 +1275,9 @@ class ScrobbleDetailView(DetailView): if data.get("with_people_ids") is not None: data["with_people_ids"] = [p.id for p in data["with_people_ids"]] + if data.get("expansion_ids") is not None: + data["expansion_ids"] = [e.id for e in data["expansion_ids"]] + if data.get("mood_reason_ids") is not None: data["mood_reason_ids"] = [r.id for r in data["mood_reason_ids"]]