[boardgames] Tighten up boardgame lookups

2025-11-02 23:43:19 -05:00
parent a8747dfe77
commit dcb5260cfc
8 changed files with 234 additions and 55 deletions
--- a/PROJECT.org
+++ b/PROJECT.org
@ -92,7 +92,7 @@ fetching and simple saving.
 :LOGBOOK:
 CLOCK: [2025-07-09 Wed 09:55]--[2025-07-09 Wed 10:15] =>  0:20
 :END:
-* Backlog [1/26]
+* Backlog [4/27]
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles:
 ** TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :utility:improvement:
 :PROPERTIES:
@ -422,11 +422,6 @@ https://app.todoist.com/app/task/add-a-csv-endpoint-for-users-book-reads-that-li

  As an example https://comicbookroundup.com/comic-books/reviews/humanoids-publishing/the-history-of-science-fiction
 ** TODO [#B] Add PuzzleLogData class with with_people and completed :vrobbler:feature:puzzles:logdata:personal:project:
-** TODO [#A] Add classmethod for metadata fetching to tracks :vrobbler:feature:music:personal:project:
-:PROPERTIES:
-:ID:       bc4b45e5-4c65-13c5-ab7b-1937d3fbf5c2
-:END:
-
 ** TODO [#A] Tasks from org-mode should properly update notes and leave them out of the body :vrobbler:bug:tasks:
 ** TODO [#A] Fix koreader scrobble imports to use DST properly :vrobbler:personal:bug:books:imports:

@ -451,7 +446,22 @@ https://life.lab.unbl.ink/scrobble/e39779c8-62a5-46a6-bdef-fb7662810dc6/start/

  This may have already been resolved ... need to just confirm it.
 ** TODO [#A] Find page numbers for comic books from ComicVine :vrobbler:feature:books:personal:project:
-** STRT [#A] Fix views for TV series where next episode is now None :vrobbler:bug:personal:videos:
+** DONE [#A] Use bgg-api for BoardGameGeek lookups :vrobbler:feature:boardgames:personal:project:
+:PROPERTIES:
+:ID:       738abb5a-c796-b16b-fe10-6e5639a0e10d
+:END:
+** DONE [#A] Add classmethod for metadata fetching to tracks :vrobbler:feature:music:personal:project:
+:PROPERTIES:
+:ID:       bc4b45e5-4c65-13c5-ab7b-1937d3fbf5c2
+:END:
+
+- Note taken on [2025-10-29 Wed 21:44]
+
+  Beyond a classmethod (which I think we have now), we need to update the flow of how we look up tracks.
+
+  It's a hot mess right now where Various Artists walks over the actual artist, and we often hit MB when we don't have to.
+
+** DONE [#A] Fix views for TV series where next episode is now None :vrobbler:bug:personal:videos:
 :PROPERTIES:
 :ID:       d7014ac4-cda6-0802-2cdf-8f66c6389fea
 :END:
--- a/poetry.lock
+++ b/poetry.lock
@ -361,6 +361,22 @@ python-dateutil = ">=2.8.2,<3.0.0"
 requests = ">=2.28.2,<3.0.0"
 typing-extensions = ">=4.7.1,<5.0.0"

+[[package]]
+name = "bgg-api"
+version = "1.1.13"
+description = "A Python API for boardgamegeek.com"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "bgg_api-1.1.13-py3-none-any.whl", hash = "sha256:6babe32ddb0ccbba7292b789770bd64ef523cab0d2cfd0a2c326cebce3e842e7"},
+    {file = "bgg_api-1.1.13.tar.gz", hash = "sha256:1e921b1d2818157418abb90d4ae7a50d8b071f1ade4ab47add1c8fdfa333e6dc"},
+]
+
+[package.dependencies]
+requests = ">=2.31.0,<3.0.0"
+requests-cache = ">=1.1.1,<2.0.0"
+
 [[package]]
 name = "billiard"
 version = "4.2.1"
@ -519,6 +535,33 @@ dev = ["CacheControl[filecache,redis]", "build", "cherrypy", "codespell[tomli]",
 filecache = ["filelock (>=3.8.0)"]
 redis = ["redis (>=2.10.5)"]

+[[package]]
+name = "cattrs"
+version = "25.2.0"
+description = "Composable complex class support for attrs and dataclasses."
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+    {file = "cattrs-25.2.0-py3-none-any.whl", hash = "sha256:539d7eedee7d2f0706e4e109182ad096d608ba84633c32c75ef3458f1d11e8f1"},
+    {file = "cattrs-25.2.0.tar.gz", hash = "sha256:f46c918e955db0177be6aa559068390f71988e877c603ae2e56c71827165cc06"},
+]
+
+[package.dependencies]
+attrs = ">=24.3.0"
+exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""}
+typing-extensions = ">=4.12.2"
+
+[package.extras]
+bson = ["pymongo (>=4.4.0)"]
+cbor2 = ["cbor2 (>=5.4.6)"]
+msgpack = ["msgpack (>=1.0.5)"]
+msgspec = ["msgspec (>=0.19.0) ; implementation_name == \"cpython\""]
+orjson = ["orjson (>=3.10.7) ; implementation_name == \"cpython\""]
+pyyaml = ["pyyaml (>=6.0)"]
+tomlkit = ["tomlkit (>=0.11.8)"]
+ujson = ["ujson (>=5.10.0)"]
+
 [[package]]
 name = "celery"
 version = "5.4.0"
@ -4308,6 +4351,37 @@ urllib3 = ">=1.21.1,<3"
 socks = ["PySocks (>=1.5.6,!=1.5.7)"]
 use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"]

+[[package]]
+name = "requests-cache"
+version = "1.2.1"
+description = "A persistent cache for python requests"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"},
+    {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"},
+]
+
+[package.dependencies]
+attrs = ">=21.2"
+cattrs = ">=22.2"
+platformdirs = ">=2.5"
+requests = ">=2.22"
+url-normalize = ">=1.4"
+urllib3 = ">=1.25.5"
+
+[package.extras]
+all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"]
+bson = ["bson (>=0.5)"]
+docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"]
+dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"]
+json = ["ujson (>=5.4)"]
+mongodb = ["pymongo (>=3)"]
+redis = ["redis (>=3)"]
+security = ["itsdangerous (>=2.0)"]
+yaml = ["pyyaml (>=6.0.1)"]
+
 [[package]]
 name = "requests-oauthlib"
 version = "2.0.0"
@ -5013,6 +5087,24 @@ tzdata = {version = "*", markers = "platform_system == \"Windows\""}
 [package.extras]
 devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"]

+[[package]]
+name = "url-normalize"
+version = "2.2.1"
+description = "URL normalization for Python"
+optional = false
+python-versions = ">=3.8"
+groups = ["main"]
+files = [
+    {file = "url_normalize-2.2.1-py3-none-any.whl", hash = "sha256:3deb687587dc91f7b25c9ae5162ffc0f057ae85d22b1e15cf5698311247f567b"},
+    {file = "url_normalize-2.2.1.tar.gz", hash = "sha256:74a540a3b6eba1d95bdc610c24f2c0141639f3ba903501e61a52a8730247ff37"},
+]
+
+[package.dependencies]
+idna = ">=3.3"
+
+[package.extras]
+dev = ["mypy", "pre-commit", "pytest", "pytest-cov", "pytest-socket", "ruff"]
+
 [[package]]
 name = "urllib3"
 version = "1.26.20"
@ -5539,4 +5631,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.9,<3.12"
-content-hash = "2e297ef6f8c524840a381ad793946c87b601d81afd569e882fe58120a5f84626"
+content-hash = "f89cff0d1019afe54e4df89a8debf50b79776c474e60d48fcae1e7c70daa3761"
--- a/pyproject.toml
+++ b/pyproject.toml
@ -58,6 +58,7 @@ tmdbv3api = "^1.9.0"
 themoviedb = "^1.0.2"
 feedparser = "^6.0.12"
 titlecase = "^2.4.1"
+bgg-api = "^1.1.13"

 [tool.poetry.group.test]
 optional = true
--- a/vrobbler/apps/boardgames/migrations/0012_boardgame_bgg_rank.py
+++ b/vrobbler/apps/boardgames/migrations/0012_boardgame_bgg_rank.py
@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-11-03 04:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boardgames', '0011_remove_boardgame_run_time_seconds_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='boardgame',
+            name='bgg_rank',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]
--- a/vrobbler/apps/boardgames/migrations/0013_boardgame_publishers.py
+++ b/vrobbler/apps/boardgames/migrations/0013_boardgame_publishers.py
@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-11-03 04:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boardgames', '0012_boardgame_bgg_rank'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='boardgame',
+            name='publishers',
+            field=models.ManyToManyField(related_name='board_games', to='boardgames.boardgamepublisher'),
+        ),
+    ]
--- a/vrobbler/apps/boardgames/models.py
+++ b/vrobbler/apps/boardgames/models.py
@ -2,12 +2,12 @@ from functools import cached_property
 import logging
 from dataclasses import dataclass
 from datetime import datetime
-from typing import Optional
+from typing import Optional, Any
 from uuid import uuid4

 from django import forms
 import requests
-from boardgames.bgg import lookup_boardgame_from_bgg
+from boardgames.sources.bgg import lookup_boardgame_from_bgg
 from django.conf import settings
 from django.core.files.base import ContentFile
 from django.db import models
@ -191,6 +191,10 @@ class BoardGame(ScrobblableMixin):
    publisher = models.ForeignKey(
        BoardGamePublisher, **BNULL, on_delete=models.DO_NOTHING
    )
+    publishers = models.ManyToManyField(
+        BoardGamePublisher,
+        related_name="board_games",
+    )
    designers = models.ManyToManyField(
        BoardGameDesigner,
        related_name="board_games",
@ -224,6 +228,7 @@ class BoardGame(ScrobblableMixin):
        options={"quality": 75},
    )
    rating = models.FloatField(**BNULL)
+    bgg_rank = models.IntegerField(**BNULL)
    max_players = models.PositiveSmallIntegerField(**BNULL)
    min_players = models.PositiveSmallIntegerField(**BNULL)
    published_date = models.DateField(**BNULL)
@ -301,29 +306,58 @@ class BoardGame(ScrobblableMixin):

            # Go get cover image if the URL is present
            if cover_url and not self.cover:
-                headers = {"User-Agent": "Vrobbler 0.11.12"}
-                r = requests.get(cover_url, headers=headers)
-                logger.debug(r.status_code)
-                if r.status_code == 200:
-                    fname = f"{self.title}_cover_{self.uuid}.jpg"
-                    self.cover.save(fname, ContentFile(r.content), save=True)
-                    logger.debug("Loaded cover image from BGGeek")
+                self.save_image_from_url(cover_url)
+
+    def save_image_from_url(self, url):
+        headers = {"User-Agent": "Vrobbler 0.11.12"}
+        r = requests.get(url, headers=headers)
+        if r.status_code == 200:
+            fname = f"{self.title}_cover_{self.uuid}.jpg"
+            self.cover.save(fname, ContentFile(r.content), save=True)

    @classmethod
    def find_or_create(
-        cls, lookup_id: str, data: Optional[dict] = {}
-    ) -> Optional["BoardGame"]:
+        cls, lookup_id: str, data: dict[str, Any] = {}
+    ) -> "BoardGame":
        """Given a Lookup ID (either BGG or BGA ID), return a board game object"""
-        boardgame = cls.objects.filter(bggeek_id=lookup_id).first()
+        game = cls.objects.filter(bggeek_id=lookup_id).first()

-        if not data or not boardgame:
-            data = lookup_boardgame_from_bgg(lookup_id)
+        if game:
+            logger.info("Board game exists in database.", extra={"lookup_id": lookup_id, "data": data})
+            return game

-        if data and not boardgame:
-            boardgame, created = cls.objects.get_or_create(
-                title=data["title"], bggeek_id=lookup_id
-            )
-            if created:
-                boardgame.fix_metadata(data=data)
+        bgg_data = lookup_boardgame_from_bgg(data.get("name"))

-        return boardgame
+        mechanics = bgg_data.pop("mechanics", [])
+        designers = bgg_data.pop("designers", [])
+        categories = bgg_data.pop("categories", [])
+        publishers = bgg_data.pop("publishers", [])
+        cover_url = bgg_data.pop("cover_url")
+
+        game = cls.objects.create(
+            **bgg_data
+        )
+
+        game.save_image_from_url(cover_url)
+        game.cooperative = data.get("cooperative", False)
+        game.highest_wins = data.get("highestWins", True)
+        game.no_points = data.get("noPoints", False)
+        game.uses_teams = data.get("useTeams", False)
+        game.bgstats_id = data.get("uuid", None)
+        game.save()
+
+        if designers:
+            for designer_name in designers:
+                designer, created = BoardGameDesigner.objects.get_or_create(
+                    name=designer_name
+                )
+                game.designers.add(designer.id)
+
+        if publishers:
+            for name in publishers:
+                publisher, _ = BoardGamePublisher.objects.get_or_create(
+                    name=name
+                )
+                game.publishers.add(publisher)
+
+        return game
--- a/vrobbler/apps/boardgames/sources/bgg.py
+++ b/vrobbler/apps/boardgames/sources/bgg.py
@ -0,0 +1,29 @@
+from typing import Any
+from boardgamegeek import BGGClient
+
+from django.conf import settings
+
+def lookup_boardgame_from_bgg(title: str) -> dict[str, Any]:
+    game_dict = {"title": title}
+
+    bgg = BGGClient(access_token=settings.BGG_ACCESS_TOKEN)
+
+    game = bgg.game(title)
+
+    if game:
+        game_dict["description"] = game.description
+        game_dict["published_year"] = game.yearpublished
+        game_dict["cover_url"] = game.image
+        game_dict["min_players"] = game.minplayers
+        game_dict["max_players"] = game.maxplayers
+        game_dict["recommended_age"] = game.minage
+        game_dict["rating"] = game.rating_average
+        game_dict["bgg_rank"] = game.bgg_rank
+        game_dict["base_run_time_seconds"] = int(game.playingtime) * 60 if game.playingtime else None
+
+        game_dict["mechanics"] = game.mechanics
+        game_dict["categories"] = game.categories
+        game_dict["designers"] = game.designers
+        game_dict["publishers"] = game.publishers
+
+    return game_dict
--- a/vrobbler/apps/scrobbles/scrobblers.py
+++ b/vrobbler/apps/scrobbles/scrobblers.py
@ -355,29 +355,6 @@ def manual_scrobble_board_game(
    return Scrobble.create_or_update(boardgame, user_id, scrobble_dict)


-def find_and_enrich_board_game_data(game_dict: dict) -> BoardGame | None:
-    """TODO Move this to a utility somewhere"""
-    game = BoardGame.find_or_create(game_dict.get("bggId"))
-
-    if game:
-        game.cooperative = game_dict.get("cooperative", False)
-        game.highest_wins = game_dict.get("highestWins", True)
-        game.no_points = game_dict.get("noPoints", False)
-        game.uses_teams = game_dict.get("useTeams", False)
-        game.bgstats_id = game_dict.get("uuid", None)
-        if not game.rating:
-            game.rating = game_dict.get("rating") / 10
-        game.save()
-
-        if game_dict.get("designers"):
-            for designer_name in game_dict.get("designers", "").split(", "):
-                designer, created = BoardGameDesigner.objects.get_or_create(
-                    name=designer_name
-                )
-                game.designers.add(designer.id)
-    return game
-
-
 def email_scrobble_board_game(
    bgstat_data: dict[str, Any], user_id: int
 ) -> list[Scrobble]:
@ -407,11 +384,11 @@ def email_scrobble_board_game(
    log_data = {}
    for game in game_list:
        logger.info(f"Finding and enriching {game.get('name')}")
-        enriched_game = find_and_enrich_board_game_data(game)
+        game_obj = BoardGame.find_or_create(game.get("bggId"), data=game)
        if game.get("isBaseGame"):
-            base_games[game.get("id")] = enriched_game
+            base_games[game.get("id")] = game_obj
        if game.get("isExpansion"):
-            expansions[game.get("id")] = enriched_game
+            expansions[game.get("id")] = game_obj

    locations = {}
    for location_dict in bgstat_data.get("locations", []):