[boardgames] Add expansion fetching

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [1/26] :vrobbler:project:personal:
+* Backlog [2/26] :vrobbler:project:personal:
 ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles:
 :PROPERTIES:
@@ -617,7 +617,10 @@ boardgame_variant_ids assocaited with their log data, and a variant is created
 for one specific board game.
 
 
-** TODO [#A] Lookup all Expansions for a game when creating it :boardgames:
+** DONE [#A] Lookup all Expansions for a game when creating it :boardgames:
+:PROPERTIES:
+:ID:       8a84b06d-555c-4701-9058-ff364c89c198
+:END:
 
 *** Description
 

vrobbler/apps/boardgames/management/commands/fetch_expansions.py

@@ -0,0 +1,78 @@
+import logging
+
+from django.core.management.base import BaseCommand
+
+from boardgames.models import BoardGame
+from boardgames.sources.bgg import lookup_boardgame_from_bgg
+from boardgames.utils import fetch_and_link_expansions
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Fetch and link expansions for existing board games from BGG"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--commit",
+            action="store_true",
+            help="Persist changes to the database",
+        )
+        parser.add_argument(
+            "--bggeek-id",
+            type=str,
+            help="Only process a single game by BGG ID",
+        )
+
+    def handle(self, *args, **options):
+        commit = options.get("commit", False)
+        bggeek_id = options.get("bggeek_id")
+
+        games = BoardGame.objects.exclude(bggeek_id__isnull=True).exclude(
+            bggeek_id=""
+        )
+        if bggeek_id:
+            games = games.filter(bggeek_id=bggeek_id)
+
+        total = games.count()
+        self.stdout.write(f"Found {total} board games with BGG IDs")
+
+        if total == 0:
+            return
+
+        updated = 0
+        for game in games.iterator(chunk_size=100):
+            try:
+                data = lookup_boardgame_from_bgg(lookup_id=str(game.bggeek_id))
+            except Exception as e:
+                self.stdout.write(
+                    self.style.WARNING(
+                        f"Failed to fetch BGG data for {game.title} ({game.bggeek_id}): {e}"
+                    )
+                )
+                continue
+
+            expansions = data.get("expansions", [])
+            if not expansions:
+                continue
+
+            if commit:
+                fetch_and_link_expansions(game, expansions)
+                updated += 1
+                self.stdout.write(
+                    f"  Linked {len(expansions)} expansions to {game.title}"
+                )
+            else:
+                self.stdout.write(
+                    f"  Would link {len(expansions)} expansions to {game.title}"
+                )
+                updated += 1
+
+        if commit:
+            self.stdout.write(
+                self.style.SUCCESS(f"Updated {updated} games (changes committed)")
+            )
+        else:
+            self.stdout.write(
+                f"Would update {updated} games (pass --commit to persist)"
+            )

vrobbler/apps/boardgames/models.py

@@ -71,7 +71,7 @@ class BoardGameLogData(BaseLogData, LongPlayLogData):
     difficulty: Optional[int] = None
     solo: Optional[bool] = None
     two_handed: Optional[bool] = None
-    expansion_ids: Optional[int] = None
+    expansion_ids: Optional[list[int]] = None
     moves: Optional[list] = None
     rated: Optional[str] = None
     speed: Optional[str] = None
@@ -112,6 +112,13 @@ class BoardGameLogData(BaseLogData, LongPlayLogData):
                 required=False,
                 widget=forms.SelectMultiple(attrs={"size": 5}),
             ),
+            "expansion_ids": forms.ModelMultipleChoiceField(
+                queryset=BoardGame.objects.filter(
+                    expansion_for_boardgame__isnull=False
+                ),
+                required=False,
+                widget=forms.SelectMultiple(attrs={"size": 5}),
+            ),
         }
         fields.update(custom_fields)
         return fields
@@ -326,18 +333,28 @@ class BoardGame(ScrobblableMixin):
             if not data:
                 data = lookup_boardgame_from_bgg(str(self.bggeek_id))
 
-            cover_url = data.pop("cover_url")
+            expansions = data.pop("expansions", [])
-            year = data.pop("year_published")
+            cover_url = data.pop("cover_url", "")
-            publisher_name = data.pop("publisher_name")
+            year_published = data.pop("year_published", None)
+            if year_published is None:
+                year_published = data.pop("published_year", None)
+            publisher_name = data.pop("publisher_name", "")
 
-            if year:
+            if year_published:
-                data["published_year"] = int(year)
+                data["published_year"] = int(year_published)
 
             if not data["min_players"]:
                 data.pop("min_players")
             if not data["min_players"]:
                 data.pop("max_players")
 
+            # Pop extra BGG metadata that isn't a model field
+            data.pop("mechanics", None)
+            data.pop("categories", None)
+            data.pop("designers", None)
+            data.pop("publishers", None)
+            data.pop("publisher", None)
+
             # Fun trick for updating all fields at once
             BoardGame.objects.filter(pk=self.id).update(**data)
             self.refresh_from_db()
@@ -353,6 +370,10 @@ class BoardGame(ScrobblableMixin):
             if cover_url and not self.cover:
                 self.save_image_from_url(cover_url)
 
+            from boardgames.utils import fetch_and_link_expansions
+
+            fetch_and_link_expansions(self, expansions)
+
     def save_image_from_url(self, url):
         headers = {"User-Agent": "Vrobbler 0.11.12"}
         r = requests.get(url, headers=headers)
@@ -372,21 +393,7 @@ class BoardGame(ScrobblableMixin):
                 "Board game exists in database.",
                 extra={"lookup_id": lookup_id, "data": data},
             )
-        return game
+            return game
-
-
-class BoardGameVariant(TimeStampedModel):
-    name = models.CharField(max_length=255)
-    board_game = models.ForeignKey(
-        BoardGame,
-        on_delete=models.CASCADE,
-        related_name="variants",
-    )
-    description = models.TextField(**BNULL)
-    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
-
-    def __str__(self) -> str:
-        return f"{self.name} ({self.board_game.title})"
 
         if data.get("bggId"):
             bgg_data = lookup_boardgame_from_bgg(lookup_id=data.get("bggId"))
@@ -398,6 +405,7 @@ class BoardGameVariant(TimeStampedModel):
             else:
                 bgg_data = lookup_boardgame_from_bgg(title=lookup_id)
 
+        expansions = bgg_data.pop("expansions", [])
         mechanics = bgg_data.pop("mechanics", [])
         designers = bgg_data.pop("designers", [])
         categories = bgg_data.pop("categories", [])
@@ -430,4 +438,22 @@ class BoardGameVariant(TimeStampedModel):
                 publisher, _ = BoardGamePublisher.objects.get_or_create(name=name)
                 game.publishers.add(publisher)
 
+        from boardgames.utils import fetch_and_link_expansions
+
+        fetch_and_link_expansions(game, expansions)
+
         return game
+
+
+class BoardGameVariant(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    board_game = models.ForeignKey(
+        BoardGame,
+        on_delete=models.CASCADE,
+        related_name="variants",
+    )
+    description = models.TextField(**BNULL)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+
+    def __str__(self) -> str:
+        return f"{self.name} ({self.board_game.title})"

vrobbler/apps/boardgames/sources/bgg.py

@@ -36,5 +36,8 @@ def lookup_boardgame_from_bgg(
         game_dict["publishers"] = game.publishers
         if game.publishers:
             game_dict["publisher"] = game.publishers[0]
+        game_dict["expansions"] = [
+            {"id": exp.id, "name": exp.name} for exp in game.expansions
+        ]
 
     return game_dict

vrobbler/apps/boardgames/tests/test_models.py

@@ -1,3 +1,5 @@
+from unittest.mock import patch
+
 import pytest
 from django.contrib.auth import get_user_model
 
@@ -63,3 +65,161 @@ def test_board_game_variant_cascade_delete():
     )
     game.delete()
     assert BoardGameVariant.objects.count() == 0
+
+
+def _mock_bgg_game(bggeek_id, title, expansions=None):
+    """Build a fake BGG game object shape used by lookup_boardgame_from_bgg."""
+    class FakeGame:
+        id = bggeek_id
+        name = title
+        description = f"Description of {title}"
+        yearpublished = 2020
+        image = "https://example.com/cover.jpg"
+        minplayers = 1
+        maxplayers = 4
+        minage = 8
+        rating_average = 7.5
+        bgg_rank = 100
+        playingtime = 60
+        mechanics = []
+        categories = []
+        designers = []
+        publishers = []
+
+        @property
+        def expansions(self):
+            if expansions is None:
+                return []
+            return expansions
+
+    return FakeGame()
+
+
+def _mock_bgg_client(return_game):
+    """Return a callable that creates a fake BGGClient instance."""
+    class FakeBGGClient:
+        def __init__(self, access_token=None):
+            pass
+
+        def game(self, game_id=None, name=None):
+            return return_game
+
+    return FakeBGGClient()
+
+
+@pytest.mark.django_db
+@patch("boardgames.models.requests.get")
+@patch("boardgames.sources.bgg.BGGClient")
+def test_find_or_create_links_expansions(mock_bgg, mock_get):
+    exp1 = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
+    exp2 = type("Thing", (), {"id": 201, "name": "Expansion 2"})()
+    base_game = _mock_bgg_game("100", "Base Game", expansions=[exp1, exp2])
+    exp1_game = _mock_bgg_game("200", "Expansion 1")
+    exp2_game = _mock_bgg_game("201", "Expansion 2")
+    mock_bgg.side_effect = [
+        _mock_bgg_client(base_game),
+        _mock_bgg_client(exp1_game),
+        _mock_bgg_client(exp2_game),
+    ]
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.content = b"fake_image_data"
+
+    game = BoardGame.find_or_create("100")
+    assert game.title == "Base Game"
+
+    expansions = BoardGame.objects.filter(expansion_for_boardgame=game)
+    assert expansions.count() == 2
+    assert {e.title for e in expansions} == {"Expansion 1", "Expansion 2"}
+
+
+@pytest.mark.django_db
+@patch("boardgames.sources.bgg.BGGClient")
+def test_find_or_create_skips_expansions_for_existing_game(mock_bgg):
+    game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
+    mock_bgg.assert_not_called()
+
+    result = BoardGame.find_or_create("100")
+    assert result == game
+
+
+@pytest.mark.django_db
+@patch("boardgames.models.requests.get")
+@patch("boardgames.sources.bgg.BGGClient")
+def test_fetch_and_link_expansions(mock_bgg, mock_get):
+    game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
+    exp_game = _mock_bgg_game("200", "Expansion 1")
+    mock_bgg.side_effect = [_mock_bgg_client(exp_game)]
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.content = b"fake_image_data"
+
+    from boardgames.utils import fetch_and_link_expansions
+
+    fetch_and_link_expansions(game, [{"id": 200, "name": "Expansion 1"}])
+
+    expansion = BoardGame.objects.get(bggeek_id="200")
+    assert expansion.expansion_for_boardgame == game
+
+
+@pytest.mark.django_db
+@patch("boardgames.models.requests.get")
+@patch("boardgames.sources.bgg.BGGClient")
+def test_fix_metadata_links_expansions(mock_bgg, mock_get):
+    game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
+    exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
+    base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])
+    exp_game = _mock_bgg_game("200", "Expansion 1")
+    # first call = fix_metadata -> lookup_boardgame_from_bgg(lookup_id='100')
+    # second call = find_or_create inside fetch_and_link_expansions for expansion
+    mock_bgg.side_effect = [
+        _mock_bgg_client(base_with_exp),
+        _mock_bgg_client(exp_game),
+    ]
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.content = b"fake_image_data"
+
+    game.fix_metadata(force_update=True)
+
+    expansion = BoardGame.objects.get(bggeek_id="200")
+    assert expansion.expansion_for_boardgame == game
+
+
+@pytest.mark.django_db
+@patch("boardgames.sources.bgg.BGGClient")
+def test_management_command_fetch_expansions_dry_run(mock_bgg, capsys):
+    from django.core.management import call_command
+
+    game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
+    exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
+    base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])
+    mock_bgg.side_effect = [_mock_bgg_client(base_with_exp)]
+
+    call_command("fetch_expansions")
+    captured = capsys.readouterr()
+    assert "Would link 1 expansions" in captured.out
+
+    assert BoardGame.objects.filter(bggeek_id="200").count() == 0
+
+
+@pytest.mark.django_db
+@patch("boardgames.models.requests.get")
+@patch("boardgames.sources.bgg.BGGClient")
+def test_management_command_fetch_expansions_commit(mock_bgg, mock_get, capsys):
+    from django.core.management import call_command
+
+    game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
+    exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
+    exp_game = _mock_bgg_game("200", "Expansion 1")
+    base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])
+    mock_bgg.side_effect = [
+        _mock_bgg_client(base_with_exp),
+        _mock_bgg_client(exp_game),
+    ]
+    mock_get.return_value.status_code = 200
+    mock_get.return_value.content = b"fake_image_data"
+
+    call_command("fetch_expansions", commit=True)
+    captured = capsys.readouterr()
+    assert "Updated 1 games" in captured.out
+
+    expansion = BoardGame.objects.get(bggeek_id="200")
+    assert expansion.expansion_for_boardgame == game

vrobbler/apps/boardgames/utils.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any
 
 from boardgames.models import BoardGame, BoardGameVariant
 
@@ -35,3 +36,27 @@ def board_names_to_variants(
             )
             variant_ids.append(variant.id)
     return variant_ids
+
+
+def fetch_and_link_expansions(
+    board_game: BoardGame, expansions_data: list[dict[str, Any]]
+) -> None:
+    """Given a board game and a list of expansion dicts (with 'id' and 'name'),
+    find or create each expansion BoardGame and link it via expansion_for_boardgame.
+    """
+    for exp_data in expansions_data:
+        exp_id = exp_data.get("id")
+        if not exp_id:
+            continue
+        expansion = BoardGame.find_or_create(str(exp_id))
+        if expansion and expansion.id != board_game.id:
+            expansion.expansion_for_boardgame = board_game
+            expansion.save(update_fields=["expansion_for_boardgame"])
+            logger.info(
+                "Linked expansion to board game",
+                extra={
+                    "board_game_id": board_game.id,
+                    "expansion_id": expansion.id,
+                    "expansion_name": expansion.title,
+                },
+            )

vrobbler/apps/scrobbles/views.py

@@ -1235,6 +1235,15 @@ class ScrobbleDetailView(DetailView):
     def get_form_class(self):
         return self.object.media_obj.logdata_cls().form()
 
+    def _update_expansion_ids_queryset(self, form):
+        from boardgames.models import BoardGame
+
+        if isinstance(self.object.media_obj, BoardGame) and "expansion_ids" in form.fields:
+            expansions = BoardGame.objects.filter(
+                expansion_for_boardgame=self.object.media_obj
+            )
+            form.fields["expansion_ids"].queryset = expansions
+
     def get_form(self):
         FormClass = self.get_form_class()
 
@@ -1245,12 +1254,15 @@ class ScrobbleDetailView(DetailView):
         else:
             log["notes"] = self.object.logdata.notes_as_str(separator="\n")
 
-        return FormClass(initial=log)
+        form = FormClass(initial=log)
+        self._update_expansion_ids_queryset(form)
+        return form
 
     def post(self, request, *args, **kwargs):
         self.object = self.get_object()
         FormClass = self.get_form_class()
         form = FormClass(request.POST)
+        self._update_expansion_ids_queryset(form)
 
         if form.is_valid():
             data = form.cleaned_data.copy()
@@ -1263,6 +1275,9 @@ class ScrobbleDetailView(DetailView):
             if data.get("with_people_ids") is not None:
                 data["with_people_ids"] = [p.id for p in data["with_people_ids"]]
 
+            if data.get("expansion_ids") is not None:
+                data["expansion_ids"] = [e.id for e in data["expansion_ids"]]
+
             if data.get("mood_reason_ids") is not None:
                 data["mood_reason_ids"] = [r.id for r in data["mood_reason_ids"]]