secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[boardgames] Add expansion fetching

Browse Source
This commit is contained in:
Colin Powell
2026-07-04 01:34:42 -04:00
parent 5aa89b7e0a
commit d5dd63be0d
7 changed files with 334 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
PROJECT.org
View File

@ -88,7 +88,7 @@ fetching and simple saving.
*** Metadata sources
**** Scraper
* Backlog [1/26] :vrobbler:project:personal:
* Backlog [2/26] :vrobbler:project:personal:
** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata:
** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles:
:PROPERTIES:
@ -617,7 +617,10 @@ boardgame_variant_ids assocaited with their log data, and a variant is created
for one specific board game.
** TODO [#A] Lookup all Expansions for a game when creating it :boardgames:
** DONE [#A] Lookup all Expansions for a game when creating it :boardgames:
:PROPERTIES:
:ID: 8a84b06d-555c-4701-9058-ff364c89c198
:END:
*** Description

78
vrobbler/apps/boardgames/management/commands/fetch_expansions.py Normal file
View File

@ -0,0 +1,78 @@
import logging
from django.core.management.base import BaseCommand
from boardgames.models import BoardGame
from boardgames.sources.bgg import lookup_boardgame_from_bgg
from boardgames.utils import fetch_and_link_expansions
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Fetch and link expansions for existing board games from BGG"
def add_arguments(self, parser):
parser.add_argument(
"--commit",
action="store_true",
help="Persist changes to the database",
)
parser.add_argument(
"--bggeek-id",
type=str,
help="Only process a single game by BGG ID",
)
def handle(self, *args, **options):
commit = options.get("commit", False)
bggeek_id = options.get("bggeek_id")
games = BoardGame.objects.exclude(bggeek_id__isnull=True).exclude(
bggeek_id=""
)
if bggeek_id:
games = games.filter(bggeek_id=bggeek_id)
total = games.count()
self.stdout.write(f"Found {total} board games with BGG IDs")
if total == 0:
return
updated = 0
for game in games.iterator(chunk_size=100):
try:
data = lookup_boardgame_from_bgg(lookup_id=str(game.bggeek_id))
except Exception as e:
self.stdout.write(
self.style.WARNING(
f"Failed to fetch BGG data for {game.title} ({game.bggeek_id}): {e}"
)
)
continue
expansions = data.get("expansions", [])
if not expansions:
continue
if commit:
fetch_and_link_expansions(game, expansions)
updated += 1
self.stdout.write(
f" Linked {len(expansions)} expansions to {game.title}"
)
else:
self.stdout.write(
f" Would link {len(expansions)} expansions to {game.title}"
)
updated += 1
if commit:
self.stdout.write(
self.style.SUCCESS(f"Updated {updated} games (changes committed)")
)
else:
self.stdout.write(
f"Would update {updated} games (pass --commit to persist)"
)

68
vrobbler/apps/boardgames/models.py
View File

@ -71,7 +71,7 @@ class BoardGameLogData(BaseLogData, LongPlayLogData):
difficulty: Optional[int] = None
solo: Optional[bool] = None
two_handed: Optional[bool] = None
expansion_ids: Optional[int] = None
expansion_ids: Optional[list[int]] = None
moves: Optional[list] = None
rated: Optional[str] = None
speed: Optional[str] = None
@ -112,6 +112,13 @@ class BoardGameLogData(BaseLogData, LongPlayLogData):
required=False,
widget=forms.SelectMultiple(attrs={"size": 5}),
),
"expansion_ids": forms.ModelMultipleChoiceField(
queryset=BoardGame.objects.filter(
expansion_for_boardgame__isnull=False
),
required=False,
widget=forms.SelectMultiple(attrs={"size": 5}),
),
}
fields.update(custom_fields)
return fields
@ -326,18 +333,28 @@ class BoardGame(ScrobblableMixin):
if not data:
data = lookup_boardgame_from_bgg(str(self.bggeek_id))
cover_url = data.pop("cover_url")
year = data.pop("year_published")
publisher_name = data.pop("publisher_name")
expansions = data.pop("expansions", [])
cover_url = data.pop("cover_url", "")
year_published = data.pop("year_published", None)
if year_published is None:
year_published = data.pop("published_year", None)
publisher_name = data.pop("publisher_name", "")
if year:
data["published_year"] = int(year)
if year_published:
data["published_year"] = int(year_published)
if not data["min_players"]:
data.pop("min_players")
if not data["min_players"]:
data.pop("max_players")
# Pop extra BGG metadata that isn't a model field
data.pop("mechanics", None)
data.pop("categories", None)
data.pop("designers", None)
data.pop("publishers", None)
data.pop("publisher", None)
# Fun trick for updating all fields at once
BoardGame.objects.filter(pk=self.id).update(**data)
self.refresh_from_db()
@ -353,6 +370,10 @@ class BoardGame(ScrobblableMixin):
if cover_url and not self.cover:
self.save_image_from_url(cover_url)
from boardgames.utils import fetch_and_link_expansions
fetch_and_link_expansions(self, expansions)
def save_image_from_url(self, url):
headers = {"User-Agent": "Vrobbler 0.11.12"}
r = requests.get(url, headers=headers)
@ -372,21 +393,7 @@ class BoardGame(ScrobblableMixin):
"Board game exists in database.",
extra={"lookup_id": lookup_id, "data": data},
)
return game
class BoardGameVariant(TimeStampedModel):
name = models.CharField(max_length=255)
board_game = models.ForeignKey(
BoardGame,
on_delete=models.CASCADE,
related_name="variants",
)
description = models.TextField(**BNULL)
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
def __str__(self) -> str:
return f"{self.name} ({self.board_game.title})"
return game
if data.get("bggId"):
bgg_data = lookup_boardgame_from_bgg(lookup_id=data.get("bggId"))
@ -398,6 +405,7 @@ class BoardGameVariant(TimeStampedModel):
else:
bgg_data = lookup_boardgame_from_bgg(title=lookup_id)
expansions = bgg_data.pop("expansions", [])
mechanics = bgg_data.pop("mechanics", [])
designers = bgg_data.pop("designers", [])
categories = bgg_data.pop("categories", [])
@ -430,4 +438,22 @@ class BoardGameVariant(TimeStampedModel):
publisher, _ = BoardGamePublisher.objects.get_or_create(name=name)
game.publishers.add(publisher)
from boardgames.utils import fetch_and_link_expansions
fetch_and_link_expansions(game, expansions)
return game
class BoardGameVariant(TimeStampedModel):
name = models.CharField(max_length=255)
board_game = models.ForeignKey(
BoardGame,
on_delete=models.CASCADE,
related_name="variants",
)
description = models.TextField(**BNULL)
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
def __str__(self) -> str:
return f"{self.name} ({self.board_game.title})"

3
vrobbler/apps/boardgames/sources/bgg.py
View File

@ -36,5 +36,8 @@ def lookup_boardgame_from_bgg(
game_dict["publishers"] = game.publishers
if game.publishers:
game_dict["publisher"] = game.publishers[0]
game_dict["expansions"] = [
{"id": exp.id, "name": exp.name} for exp in game.expansions
]
return game_dict

160
vrobbler/apps/boardgames/tests/test_models.py
View File

@ -1,3 +1,5 @@
from unittest.mock import patch
import pytest
from django.contrib.auth import get_user_model
@ -63,3 +65,161 @@ def test_board_game_variant_cascade_delete():
)
game.delete()
assert BoardGameVariant.objects.count() == 0
def _mock_bgg_game(bggeek_id, title, expansions=None):
"""Build a fake BGG game object shape used by lookup_boardgame_from_bgg."""
class FakeGame:
id = bggeek_id
name = title
description = f"Description of {title}"
yearpublished = 2020
image = "https://example.com/cover.jpg"
minplayers = 1
maxplayers = 4
minage = 8
rating_average = 7.5
bgg_rank = 100
playingtime = 60
mechanics = []
categories = []
designers = []
publishers = []
@property
def expansions(self):
if expansions is None:
return []
return expansions
return FakeGame()
def _mock_bgg_client(return_game):
"""Return a callable that creates a fake BGGClient instance."""
class FakeBGGClient:
def __init__(self, access_token=None):
pass
def game(self, game_id=None, name=None):
return return_game
return FakeBGGClient()
@pytest.mark.django_db
@patch("boardgames.models.requests.get")
@patch("boardgames.sources.bgg.BGGClient")
def test_find_or_create_links_expansions(mock_bgg, mock_get):
exp1 = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
exp2 = type("Thing", (), {"id": 201, "name": "Expansion 2"})()
base_game = _mock_bgg_game("100", "Base Game", expansions=[exp1, exp2])
exp1_game = _mock_bgg_game("200", "Expansion 1")
exp2_game = _mock_bgg_game("201", "Expansion 2")
mock_bgg.side_effect = [
_mock_bgg_client(base_game),
_mock_bgg_client(exp1_game),
_mock_bgg_client(exp2_game),
]
mock_get.return_value.status_code = 200
mock_get.return_value.content = b"fake_image_data"
game = BoardGame.find_or_create("100")
assert game.title == "Base Game"
expansions = BoardGame.objects.filter(expansion_for_boardgame=game)
assert expansions.count() == 2
assert {e.title for e in expansions} == {"Expansion 1", "Expansion 2"}
@pytest.mark.django_db
@patch("boardgames.sources.bgg.BGGClient")
def test_find_or_create_skips_expansions_for_existing_game(mock_bgg):
game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
mock_bgg.assert_not_called()
result = BoardGame.find_or_create("100")
assert result == game
@pytest.mark.django_db
@patch("boardgames.models.requests.get")
@patch("boardgames.sources.bgg.BGGClient")
def test_fetch_and_link_expansions(mock_bgg, mock_get):
game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
exp_game = _mock_bgg_game("200", "Expansion 1")
mock_bgg.side_effect = [_mock_bgg_client(exp_game)]
mock_get.return_value.status_code = 200
mock_get.return_value.content = b"fake_image_data"
from boardgames.utils import fetch_and_link_expansions
fetch_and_link_expansions(game, [{"id": 200, "name": "Expansion 1"}])
expansion = BoardGame.objects.get(bggeek_id="200")
assert expansion.expansion_for_boardgame == game
@pytest.mark.django_db
@patch("boardgames.models.requests.get")
@patch("boardgames.sources.bgg.BGGClient")
def test_fix_metadata_links_expansions(mock_bgg, mock_get):
game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])
exp_game = _mock_bgg_game("200", "Expansion 1")
# first call = fix_metadata -> lookup_boardgame_from_bgg(lookup_id='100')
# second call = find_or_create inside fetch_and_link_expansions for expansion
mock_bgg.side_effect = [
_mock_bgg_client(base_with_exp),
_mock_bgg_client(exp_game),
]
mock_get.return_value.status_code = 200
mock_get.return_value.content = b"fake_image_data"
game.fix_metadata(force_update=True)
expansion = BoardGame.objects.get(bggeek_id="200")
assert expansion.expansion_for_boardgame == game
@pytest.mark.django_db
@patch("boardgames.sources.bgg.BGGClient")
def test_management_command_fetch_expansions_dry_run(mock_bgg, capsys):
from django.core.management import call_command
game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])
mock_bgg.side_effect = [_mock_bgg_client(base_with_exp)]
call_command("fetch_expansions")
captured = capsys.readouterr()
assert "Would link 1 expansions" in captured.out
assert BoardGame.objects.filter(bggeek_id="200").count() == 0
@pytest.mark.django_db
@patch("boardgames.models.requests.get")
@patch("boardgames.sources.bgg.BGGClient")
def test_management_command_fetch_expansions_commit(mock_bgg, mock_get, capsys):
from django.core.management import call_command
game = BoardGame.objects.create(title="Base Game", bggeek_id="100")
exp_thing = type("Thing", (), {"id": 200, "name": "Expansion 1"})()
exp_game = _mock_bgg_game("200", "Expansion 1")
base_with_exp = _mock_bgg_game("100", "Base Game", expansions=[exp_thing])
mock_bgg.side_effect = [
_mock_bgg_client(base_with_exp),
_mock_bgg_client(exp_game),
]
mock_get.return_value.status_code = 200
mock_get.return_value.content = b"fake_image_data"
call_command("fetch_expansions", commit=True)
captured = capsys.readouterr()
assert "Updated 1 games" in captured.out
expansion = BoardGame.objects.get(bggeek_id="200")
assert expansion.expansion_for_boardgame == game

25
vrobbler/apps/boardgames/utils.py
View File

@ -1,4 +1,5 @@
import logging
from typing import Any
from boardgames.models import BoardGame, BoardGameVariant
@ -35,3 +36,27 @@ def board_names_to_variants(
)
variant_ids.append(variant.id)
return variant_ids
def fetch_and_link_expansions(
board_game: BoardGame, expansions_data: list[dict[str, Any]]
) -> None:
"""Given a board game and a list of expansion dicts (with 'id' and 'name'),
find or create each expansion BoardGame and link it via expansion_for_boardgame.
"""
for exp_data in expansions_data:
exp_id = exp_data.get("id")
if not exp_id:
continue
expansion = BoardGame.find_or_create(str(exp_id))
if expansion and expansion.id != board_game.id:
expansion.expansion_for_boardgame = board_game
expansion.save(update_fields=["expansion_for_boardgame"])
logger.info(
"Linked expansion to board game",
extra={
"board_game_id": board_game.id,
"expansion_id": expansion.id,
"expansion_name": expansion.title,
},
)

17
vrobbler/apps/scrobbles/views.py
View File

@ -1235,6 +1235,15 @@ class ScrobbleDetailView(DetailView):
def get_form_class(self):
return self.object.media_obj.logdata_cls().form()
def _update_expansion_ids_queryset(self, form):
from boardgames.models import BoardGame
if isinstance(self.object.media_obj, BoardGame) and "expansion_ids" in form.fields:
expansions = BoardGame.objects.filter(
expansion_for_boardgame=self.object.media_obj
)
form.fields["expansion_ids"].queryset = expansions
def get_form(self):
FormClass = self.get_form_class()
@ -1245,12 +1254,15 @@ class ScrobbleDetailView(DetailView):
else:
log["notes"] = self.object.logdata.notes_as_str(separator="\n")
return FormClass(initial=log)
form = FormClass(initial=log)
self._update_expansion_ids_queryset(form)
return form
def post(self, request, *args, **kwargs):
self.object = self.get_object()
FormClass = self.get_form_class()
form = FormClass(request.POST)
self._update_expansion_ids_queryset(form)
if form.is_valid():
data = form.cleaned_data.copy()
@ -1263,6 +1275,9 @@ class ScrobbleDetailView(DetailView):
if data.get("with_people_ids") is not None:
data["with_people_ids"] = [p.id for p in data["with_people_ids"]]
if data.get("expansion_ids") is not None:
data["expansion_ids"] = [e.id for e in data["expansion_ids"]]
if data.get("mood_reason_ids") is not None:
data["mood_reason_ids"] = [r.id for r in data["mood_reason_ids"]]