diff --git a/PROJECT.org b/PROJECT.org index 0c78a46..bb58191 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [1/26] :vrobbler:project:personal: +* Backlog [2/28] :vrobbler:project:personal: ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: @@ -619,7 +619,34 @@ The Edit log form should have from top to bottom: - Expansion ids (which should a multi-select widget of expansions for this game) - Location (which should be a drop down of BoardGameLocations for this user) -** TODO Exclude some board games from auto-expansion imports :boardgames: +** TODO Board games should have genres extracted from family data :boardgames:metadata: +** TODO Fix bug in fetching expansions for board games :boardgames: +:PROPERTIES: +:ID: 17995312-e76e-4a50-b591-0eab78cb59ab +:END: + +#+begin_src python + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/vrobbler/apps/boardgames/management/commands/fetch_expansions.py", line 60, in handle + fetch_and_link_expansions(game, expansions) + File "/usr/local/lib/python3.11/site-packages/vrobbler/apps/boardgames/utils.py", line 51, in fetch_and_link_expansions + expansion = BoardGame.find_or_create(str(exp_id)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/vrobbler/apps/boardgames/models.py", line 409, in find_or_create + bgg_data = lookup_boardgame_from_bgg(lookup_id=lookup_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/vrobbler/apps/boardgames/sources/bgg.py", line 15, in lookup_boardgame_from_bgg + game = bgg.game(game_id=lookup_id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/site-packages/boardgamegeek/api.py", line 1045, in game + raise BGGApiError(msg) +boardgamegeek.exceptions.BGGApiError: invalid data for game id: 242117 +#+end_src + +** DONE Exclude some board games from auto-expansion imports :boardgames: +:PROPERTIES: +:ID: 51ffdf20-e732-4774-b781-c3501d26d46f +:END: *** Description diff --git a/vrobbler/apps/boardgames/management/commands/fetch_expansions.py b/vrobbler/apps/boardgames/management/commands/fetch_expansions.py index e1e41c6..991f588 100644 --- a/vrobbler/apps/boardgames/management/commands/fetch_expansions.py +++ b/vrobbler/apps/boardgames/management/commands/fetch_expansions.py @@ -28,8 +28,10 @@ class Command(BaseCommand): commit = options.get("commit", False) bggeek_id = options.get("bggeek_id") - games = BoardGame.objects.exclude(bggeek_id__isnull=True).exclude( - bggeek_id="" + games = ( + BoardGame.objects.exclude(bggeek_id__isnull=True) + .exclude(bggeek_id="") + .exclude(skip_expansions=True) ) if bggeek_id: games = games.filter(bggeek_id=bggeek_id) diff --git a/vrobbler/apps/boardgames/migrations/0017_boardgame_skip_expansions.py b/vrobbler/apps/boardgames/migrations/0017_boardgame_skip_expansions.py new file mode 100644 index 0000000..fed9871 --- /dev/null +++ b/vrobbler/apps/boardgames/migrations/0017_boardgame_skip_expansions.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.29 on 2026-07-04 15:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("boardgames", "0016_boardgamevariant"), + ] + + operations = [ + migrations.AddField( + model_name="boardgame", + name="skip_expansions", + field=models.BooleanField(default=False), + ), + ] diff --git a/vrobbler/apps/boardgames/models.py b/vrobbler/apps/boardgames/models.py index 3878f41..f0647b2 100644 --- a/vrobbler/apps/boardgames/models.py +++ b/vrobbler/apps/boardgames/models.py @@ -300,6 +300,7 @@ class BoardGame(ScrobblableMixin): expansion_for_boardgame = models.ForeignKey( "self", **BNULL, on_delete=models.DO_NOTHING ) + skip_expansions = models.BooleanField(default=False) @property def subtitle(self) -> str: @@ -373,7 +374,8 @@ class BoardGame(ScrobblableMixin): from boardgames.utils import fetch_and_link_expansions - fetch_and_link_expansions(self, expansions) + if not self.skip_expansions: + fetch_and_link_expansions(self, expansions) def save_image_from_url(self, url): headers = {"User-Agent": "Vrobbler 0.11.12"} @@ -431,6 +433,18 @@ class BoardGame(ScrobblableMixin): if publisher: publisher, _ = BoardGamePublisher.objects.get_or_create(name=publisher) game.publisher = publisher + + skip_expansions = ( + game.bggeek_id + and game.bggeek_id.isdigit() + and int(game.bggeek_id) in settings.SKIP_AUTO_EXPANSION_DOWNLOAD + ) or any( + c == "Collectible Card Games" for c in categories + ) + + if skip_expansions: + game.skip_expansions = True + game.save() if designers: @@ -445,14 +459,15 @@ class BoardGame(ScrobblableMixin): publisher, _ = BoardGamePublisher.objects.get_or_create(name=name) game.publishers.add(publisher) - if defer_expansions: - from boardgames.tasks import fetch_board_game_expansions + if expansions and not game.skip_expansions: + if defer_expansions: + from boardgames.tasks import fetch_board_game_expansions - fetch_board_game_expansions.delay(game.id, expansions) - else: - from boardgames.utils import fetch_and_link_expansions + fetch_board_game_expansions.delay(game.id, expansions) + else: + from boardgames.utils import fetch_and_link_expansions - fetch_and_link_expansions(game, expansions) + fetch_and_link_expansions(game, expansions) return game diff --git a/vrobbler/apps/boardgames/utils.py b/vrobbler/apps/boardgames/utils.py index 92e86a0..ad670ae 100644 --- a/vrobbler/apps/boardgames/utils.py +++ b/vrobbler/apps/boardgames/utils.py @@ -44,6 +44,13 @@ def fetch_and_link_expansions( """Given a board game and a list of expansion dicts (with 'id' and 'name'), find or create each expansion BoardGame and link it via expansion_for_boardgame. """ + if board_game.skip_expansions: + logger.info( + "Skipping expansion fetch for board game with skip_expansions=True", + extra={"board_game_id": board_game.id, "title": board_game.title}, + ) + return + for exp_data in expansions_data: exp_id = exp_data.get("id") if not exp_id: diff --git a/vrobbler/settings.py b/vrobbler/settings.py index 5dd15c2..a9d9914 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -405,6 +405,12 @@ else: SCIHUB_DOMAIN = os.getenv("VROBBLER_SCIHUB_DOMAIN", "sci-hub.st") +SKIP_AUTO_EXPANSION_DOWNLOAD = [ + int(x.strip()) + for x in os.getenv("VROBBLER_SKIP_AUTO_EXPANSION_DOWNLOAD", "").split(",") + if x.strip().isdigit() +] + JSON_LOGGING = os.getenv("VROBBLER_JSON_LOGGING", "false").lower() in TRUTHY LOG_TYPE = "json" if JSON_LOGGING else "log"