From dcb5260cfc4c29d864dfa3e5b397a272c7fca382 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Sun, 2 Nov 2025 23:43:19 -0500 Subject: [PATCH] [boardgames] Tighten up boardgame lookups --- PROJECT.org | 24 +++-- poetry.lock | 94 ++++++++++++++++++- pyproject.toml | 1 + .../migrations/0012_boardgame_bgg_rank.py | 18 ++++ .../migrations/0013_boardgame_publishers.py | 18 ++++ vrobbler/apps/boardgames/models.py | 76 ++++++++++----- vrobbler/apps/boardgames/sources/bgg.py | 29 ++++++ vrobbler/apps/scrobbles/scrobblers.py | 29 +----- 8 files changed, 234 insertions(+), 55 deletions(-) create mode 100644 vrobbler/apps/boardgames/migrations/0012_boardgame_bgg_rank.py create mode 100644 vrobbler/apps/boardgames/migrations/0013_boardgame_publishers.py create mode 100644 vrobbler/apps/boardgames/sources/bgg.py diff --git a/PROJECT.org b/PROJECT.org index 485bbae..cc177f2 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -92,7 +92,7 @@ fetching and simple saving. :LOGBOOK: CLOCK: [2025-07-09 Wed 09:55]--[2025-07-09 Wed 10:15] => 0:20 :END: -* Backlog [1/26] +* Backlog [4/27] ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: ** TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :utility:improvement: :PROPERTIES: @@ -422,11 +422,6 @@ https://app.todoist.com/app/task/add-a-csv-endpoint-for-users-book-reads-that-li As an example https://comicbookroundup.com/comic-books/reviews/humanoids-publishing/the-history-of-science-fiction ** TODO [#B] Add PuzzleLogData class with with_people and completed :vrobbler:feature:puzzles:logdata:personal:project: -** TODO [#A] Add classmethod for metadata fetching to tracks :vrobbler:feature:music:personal:project: -:PROPERTIES: -:ID: bc4b45e5-4c65-13c5-ab7b-1937d3fbf5c2 -:END: - ** TODO [#A] Tasks from org-mode should properly update notes and leave them out of the body :vrobbler:bug:tasks: ** TODO [#A] Fix koreader scrobble imports to use DST properly :vrobbler:personal:bug:books:imports: @@ -451,7 +446,22 @@ https://life.lab.unbl.ink/scrobble/e39779c8-62a5-46a6-bdef-fb7662810dc6/start/ This may have already been resolved ... need to just confirm it. ** TODO [#A] Find page numbers for comic books from ComicVine :vrobbler:feature:books:personal:project: -** STRT [#A] Fix views for TV series where next episode is now None :vrobbler:bug:personal:videos: +** DONE [#A] Use bgg-api for BoardGameGeek lookups :vrobbler:feature:boardgames:personal:project: +:PROPERTIES: +:ID: 738abb5a-c796-b16b-fe10-6e5639a0e10d +:END: +** DONE [#A] Add classmethod for metadata fetching to tracks :vrobbler:feature:music:personal:project: +:PROPERTIES: +:ID: bc4b45e5-4c65-13c5-ab7b-1937d3fbf5c2 +:END: + +- Note taken on [2025-10-29 Wed 21:44] + + Beyond a classmethod (which I think we have now), we need to update the flow of how we look up tracks. + + It's a hot mess right now where Various Artists walks over the actual artist, and we often hit MB when we don't have to. + +** DONE [#A] Fix views for TV series where next episode is now None :vrobbler:bug:personal:videos: :PROPERTIES: :ID: d7014ac4-cda6-0802-2cdf-8f66c6389fea :END: diff --git a/poetry.lock b/poetry.lock index 40aae03..ee5aa7c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -361,6 +361,22 @@ python-dateutil = ">=2.8.2,<3.0.0" requests = ">=2.28.2,<3.0.0" typing-extensions = ">=4.7.1,<5.0.0" +[[package]] +name = "bgg-api" +version = "1.1.13" +description = "A Python API for boardgamegeek.com" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "bgg_api-1.1.13-py3-none-any.whl", hash = "sha256:6babe32ddb0ccbba7292b789770bd64ef523cab0d2cfd0a2c326cebce3e842e7"}, + {file = "bgg_api-1.1.13.tar.gz", hash = "sha256:1e921b1d2818157418abb90d4ae7a50d8b071f1ade4ab47add1c8fdfa333e6dc"}, +] + +[package.dependencies] +requests = ">=2.31.0,<3.0.0" +requests-cache = ">=1.1.1,<2.0.0" + [[package]] name = "billiard" version = "4.2.1" @@ -519,6 +535,33 @@ dev = ["CacheControl[filecache,redis]", "build", "cherrypy", "codespell[tomli]", filecache = ["filelock (>=3.8.0)"] redis = ["redis (>=2.10.5)"] +[[package]] +name = "cattrs" +version = "25.2.0" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "cattrs-25.2.0-py3-none-any.whl", hash = "sha256:539d7eedee7d2f0706e4e109182ad096d608ba84633c32c75ef3458f1d11e8f1"}, + {file = "cattrs-25.2.0.tar.gz", hash = "sha256:f46c918e955db0177be6aa559068390f71988e877c603ae2e56c71827165cc06"}, +] + +[package.dependencies] +attrs = ">=24.3.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = ">=4.12.2" + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +msgspec = ["msgspec (>=0.19.0) ; implementation_name == \"cpython\""] +orjson = ["orjson (>=3.10.7) ; implementation_name == \"cpython\""] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.10.0)"] + [[package]] name = "celery" version = "5.4.0" @@ -4308,6 +4351,37 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-cache" +version = "1.2.1" +description = "A persistent cache for python requests" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "requests_cache-1.2.1-py3-none-any.whl", hash = "sha256:1285151cddf5331067baa82598afe2d47c7495a1334bfe7a7d329b43e9fd3603"}, + {file = "requests_cache-1.2.1.tar.gz", hash = "sha256:68abc986fdc5b8d0911318fbb5f7c80eebcd4d01bfacc6685ecf8876052511d1"}, +] + +[package.dependencies] +attrs = ">=21.2" +cattrs = ">=22.2" +platformdirs = ">=2.5" +requests = ">=2.22" +url-normalize = ">=1.4" +urllib3 = ">=1.25.5" + +[package.extras] +all = ["boto3 (>=1.15)", "botocore (>=1.18)", "itsdangerous (>=2.0)", "pymongo (>=3)", "pyyaml (>=6.0.1)", "redis (>=3)", "ujson (>=5.4)"] +bson = ["bson (>=0.5)"] +docs = ["furo (>=2023.3,<2024.0)", "linkify-it-py (>=2.0,<3.0)", "myst-parser (>=1.0,<2.0)", "sphinx (>=5.0.2,<6.0.0)", "sphinx-autodoc-typehints (>=1.19)", "sphinx-automodapi (>=0.14)", "sphinx-copybutton (>=0.5)", "sphinx-design (>=0.2)", "sphinx-notfound-page (>=0.8)", "sphinxcontrib-apidoc (>=0.3)", "sphinxext-opengraph (>=0.9)"] +dynamodb = ["boto3 (>=1.15)", "botocore (>=1.18)"] +json = ["ujson (>=5.4)"] +mongodb = ["pymongo (>=3)"] +redis = ["redis (>=3)"] +security = ["itsdangerous (>=2.0)"] +yaml = ["pyyaml (>=6.0.1)"] + [[package]] name = "requests-oauthlib" version = "2.0.0" @@ -5013,6 +5087,24 @@ tzdata = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"] +[[package]] +name = "url-normalize" +version = "2.2.1" +description = "URL normalization for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "url_normalize-2.2.1-py3-none-any.whl", hash = "sha256:3deb687587dc91f7b25c9ae5162ffc0f057ae85d22b1e15cf5698311247f567b"}, + {file = "url_normalize-2.2.1.tar.gz", hash = "sha256:74a540a3b6eba1d95bdc610c24f2c0141639f3ba903501e61a52a8730247ff37"}, +] + +[package.dependencies] +idna = ">=3.3" + +[package.extras] +dev = ["mypy", "pre-commit", "pytest", "pytest-cov", "pytest-socket", "ruff"] + [[package]] name = "urllib3" version = "1.26.20" @@ -5539,4 +5631,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<3.12" -content-hash = "2e297ef6f8c524840a381ad793946c87b601d81afd569e882fe58120a5f84626" +content-hash = "f89cff0d1019afe54e4df89a8debf50b79776c474e60d48fcae1e7c70daa3761" diff --git a/pyproject.toml b/pyproject.toml index 2081c15..890eeac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ tmdbv3api = "^1.9.0" themoviedb = "^1.0.2" feedparser = "^6.0.12" titlecase = "^2.4.1" +bgg-api = "^1.1.13" [tool.poetry.group.test] optional = true diff --git a/vrobbler/apps/boardgames/migrations/0012_boardgame_bgg_rank.py b/vrobbler/apps/boardgames/migrations/0012_boardgame_bgg_rank.py new file mode 100644 index 0000000..eeed411 --- /dev/null +++ b/vrobbler/apps/boardgames/migrations/0012_boardgame_bgg_rank.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.19 on 2025-11-03 04:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boardgames', '0011_remove_boardgame_run_time_seconds_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='boardgame', + name='bgg_rank', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/vrobbler/apps/boardgames/migrations/0013_boardgame_publishers.py b/vrobbler/apps/boardgames/migrations/0013_boardgame_publishers.py new file mode 100644 index 0000000..f1354f1 --- /dev/null +++ b/vrobbler/apps/boardgames/migrations/0013_boardgame_publishers.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.19 on 2025-11-03 04:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('boardgames', '0012_boardgame_bgg_rank'), + ] + + operations = [ + migrations.AddField( + model_name='boardgame', + name='publishers', + field=models.ManyToManyField(related_name='board_games', to='boardgames.boardgamepublisher'), + ), + ] diff --git a/vrobbler/apps/boardgames/models.py b/vrobbler/apps/boardgames/models.py index bbda1af..8ede19b 100644 --- a/vrobbler/apps/boardgames/models.py +++ b/vrobbler/apps/boardgames/models.py @@ -2,12 +2,12 @@ from functools import cached_property import logging from dataclasses import dataclass from datetime import datetime -from typing import Optional +from typing import Optional, Any from uuid import uuid4 from django import forms import requests -from boardgames.bgg import lookup_boardgame_from_bgg +from boardgames.sources.bgg import lookup_boardgame_from_bgg from django.conf import settings from django.core.files.base import ContentFile from django.db import models @@ -191,6 +191,10 @@ class BoardGame(ScrobblableMixin): publisher = models.ForeignKey( BoardGamePublisher, **BNULL, on_delete=models.DO_NOTHING ) + publishers = models.ManyToManyField( + BoardGamePublisher, + related_name="board_games", + ) designers = models.ManyToManyField( BoardGameDesigner, related_name="board_games", @@ -224,6 +228,7 @@ class BoardGame(ScrobblableMixin): options={"quality": 75}, ) rating = models.FloatField(**BNULL) + bgg_rank = models.IntegerField(**BNULL) max_players = models.PositiveSmallIntegerField(**BNULL) min_players = models.PositiveSmallIntegerField(**BNULL) published_date = models.DateField(**BNULL) @@ -301,29 +306,58 @@ class BoardGame(ScrobblableMixin): # Go get cover image if the URL is present if cover_url and not self.cover: - headers = {"User-Agent": "Vrobbler 0.11.12"} - r = requests.get(cover_url, headers=headers) - logger.debug(r.status_code) - if r.status_code == 200: - fname = f"{self.title}_cover_{self.uuid}.jpg" - self.cover.save(fname, ContentFile(r.content), save=True) - logger.debug("Loaded cover image from BGGeek") + self.save_image_from_url(cover_url) + + def save_image_from_url(self, url): + headers = {"User-Agent": "Vrobbler 0.11.12"} + r = requests.get(url, headers=headers) + if r.status_code == 200: + fname = f"{self.title}_cover_{self.uuid}.jpg" + self.cover.save(fname, ContentFile(r.content), save=True) @classmethod def find_or_create( - cls, lookup_id: str, data: Optional[dict] = {} - ) -> Optional["BoardGame"]: + cls, lookup_id: str, data: dict[str, Any] = {} + ) -> "BoardGame": """Given a Lookup ID (either BGG or BGA ID), return a board game object""" - boardgame = cls.objects.filter(bggeek_id=lookup_id).first() + game = cls.objects.filter(bggeek_id=lookup_id).first() - if not data or not boardgame: - data = lookup_boardgame_from_bgg(lookup_id) + if game: + logger.info("Board game exists in database.", extra={"lookup_id": lookup_id, "data": data}) + return game - if data and not boardgame: - boardgame, created = cls.objects.get_or_create( - title=data["title"], bggeek_id=lookup_id - ) - if created: - boardgame.fix_metadata(data=data) + bgg_data = lookup_boardgame_from_bgg(data.get("name")) - return boardgame + mechanics = bgg_data.pop("mechanics", []) + designers = bgg_data.pop("designers", []) + categories = bgg_data.pop("categories", []) + publishers = bgg_data.pop("publishers", []) + cover_url = bgg_data.pop("cover_url") + + game = cls.objects.create( + **bgg_data + ) + + game.save_image_from_url(cover_url) + game.cooperative = data.get("cooperative", False) + game.highest_wins = data.get("highestWins", True) + game.no_points = data.get("noPoints", False) + game.uses_teams = data.get("useTeams", False) + game.bgstats_id = data.get("uuid", None) + game.save() + + if designers: + for designer_name in designers: + designer, created = BoardGameDesigner.objects.get_or_create( + name=designer_name + ) + game.designers.add(designer.id) + + if publishers: + for name in publishers: + publisher, _ = BoardGamePublisher.objects.get_or_create( + name=name + ) + game.publishers.add(publisher) + + return game diff --git a/vrobbler/apps/boardgames/sources/bgg.py b/vrobbler/apps/boardgames/sources/bgg.py new file mode 100644 index 0000000..ee77e76 --- /dev/null +++ b/vrobbler/apps/boardgames/sources/bgg.py @@ -0,0 +1,29 @@ +from typing import Any +from boardgamegeek import BGGClient + +from django.conf import settings + +def lookup_boardgame_from_bgg(title: str) -> dict[str, Any]: + game_dict = {"title": title} + + bgg = BGGClient(access_token=settings.BGG_ACCESS_TOKEN) + + game = bgg.game(title) + + if game: + game_dict["description"] = game.description + game_dict["published_year"] = game.yearpublished + game_dict["cover_url"] = game.image + game_dict["min_players"] = game.minplayers + game_dict["max_players"] = game.maxplayers + game_dict["recommended_age"] = game.minage + game_dict["rating"] = game.rating_average + game_dict["bgg_rank"] = game.bgg_rank + game_dict["base_run_time_seconds"] = int(game.playingtime) * 60 if game.playingtime else None + + game_dict["mechanics"] = game.mechanics + game_dict["categories"] = game.categories + game_dict["designers"] = game.designers + game_dict["publishers"] = game.publishers + + return game_dict diff --git a/vrobbler/apps/scrobbles/scrobblers.py b/vrobbler/apps/scrobbles/scrobblers.py index 4120b35..e6f2ee3 100644 --- a/vrobbler/apps/scrobbles/scrobblers.py +++ b/vrobbler/apps/scrobbles/scrobblers.py @@ -355,29 +355,6 @@ def manual_scrobble_board_game( return Scrobble.create_or_update(boardgame, user_id, scrobble_dict) -def find_and_enrich_board_game_data(game_dict: dict) -> BoardGame | None: - """TODO Move this to a utility somewhere""" - game = BoardGame.find_or_create(game_dict.get("bggId")) - - if game: - game.cooperative = game_dict.get("cooperative", False) - game.highest_wins = game_dict.get("highestWins", True) - game.no_points = game_dict.get("noPoints", False) - game.uses_teams = game_dict.get("useTeams", False) - game.bgstats_id = game_dict.get("uuid", None) - if not game.rating: - game.rating = game_dict.get("rating") / 10 - game.save() - - if game_dict.get("designers"): - for designer_name in game_dict.get("designers", "").split(", "): - designer, created = BoardGameDesigner.objects.get_or_create( - name=designer_name - ) - game.designers.add(designer.id) - return game - - def email_scrobble_board_game( bgstat_data: dict[str, Any], user_id: int ) -> list[Scrobble]: @@ -407,11 +384,11 @@ def email_scrobble_board_game( log_data = {} for game in game_list: logger.info(f"Finding and enriching {game.get('name')}") - enriched_game = find_and_enrich_board_game_data(game) + game_obj = BoardGame.find_or_create(game.get("bggId"), data=game) if game.get("isBaseGame"): - base_games[game.get("id")] = enriched_game + base_games[game.get("id")] = game_obj if game.get("isExpansion"): - expansions[game.get("id")] = enriched_game + expansions[game.get("id")] = game_obj locations = {} for location_dict in bgstat_data.get("locations", []):