secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[boardgames] Add lichess importing

Browse Source
This commit is contained in:
Colin Powell
2025-01-29 00:04:49 -05:00
parent fd726a125f
commit 9c115c0b65
9 changed files with 215 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
poetry.lock generated
View File

@ -294,6 +294,24 @@ charset-normalizer = ["charset-normalizer"]
html5lib = ["html5lib"]
lxml = ["lxml"]
[[package]]
name = "berserk"
version = "0.13.2"
description = "Python client for the lichess API"
optional = false
python-versions = ">=3.8,<4.0"
files = [
{file = "berserk-0.13.2-py3-none-any.whl", hash = "sha256:0f7fc40f152370924cb05a77c3f1c357a91e8ff0db60d23c14f0f16216b632a8"},
{file = "berserk-0.13.2.tar.gz", hash = "sha256:96c3ff3a10407842019e5e6bf3233080030419e4eba333bbd4234a86b4eff86f"},
]
[package.dependencies]
deprecated = ">=1.2.14,<2.0.0"
ndjson = ">=0.3.1,<0.4.0"
python-dateutil = ">=2.8.2,<3.0.0"
requests = ">=2.28.2,<3.0.0"
typing-extensions = ">=4.7.1,<5.0.0"
[[package]]
name = "billiard"
version = "4.2.1"
@ -360,17 +378,17 @@ css = ["tinycss2 (>=1.1.0,<1.5)"]
[[package]]
name = "boto3"
version = "1.36.3"
version = "1.36.8"
description = "The AWS SDK for Python"
optional = false
python-versions = ">=3.8"
files = [
{file = "boto3-1.36.3-py3-none-any.whl", hash = "sha256:f9843a5d06f501d66ada06f5a5417f671823af2cf319e36ceefa1bafaaaaa953"},
{file = "boto3-1.36.3.tar.gz", hash = "sha256:53a5307f6a3526ee2f8590e3c45efa504a3ea4532c1bfe4926c0c19bf188d141"},
{file = "boto3-1.36.8-py3-none-any.whl", hash = "sha256:7f61c9d0ea64f484a17c1e3115fdf90fd7b17ab6771e07cb4549f42b9fd28fb9"},
{file = "boto3-1.36.8.tar.gz", hash = "sha256:ac47215d320b0c2534340db58d6d5284cb1860b7bff172b4dd6eee2dee1d5779"},
]
[package.dependencies]
botocore = ">=1.36.3,<1.37.0"
botocore = ">=1.36.8,<1.37.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.11.0,<0.12.0"
@ -379,13 +397,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
version = "1.36.3"
version = "1.36.8"
description = "Low-level, data-driven core of boto 3."
optional = false
python-versions = ">=3.8"
files = [
{file = "botocore-1.36.3-py3-none-any.whl", hash = "sha256:536ab828e6f90dbb000e3702ac45fd76642113ae2db1b7b1373ad24104e89255"},
{file = "botocore-1.36.3.tar.gz", hash = "sha256:775b835e979da5c96548ed1a0b798101a145aec3cd46541d62e27dda5a94d7f8"},
{file = "botocore-1.36.8-py3-none-any.whl", hash = "sha256:59d3fdfbae6d916b046e973bebcbeb70a102f9e570ca86d5ba512f1854b78fc2"},
{file = "botocore-1.36.8.tar.gz", hash = "sha256:81c88e5566cf018e1411a68304dc1fb9e4156ca2b50a3a0f0befc274299e67fa"},
]
[package.dependencies]
@ -963,6 +981,23 @@ files = [
{file = "defusedxml-0.7.1.tar.gz", hash = "sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69"},
]
[[package]]
name = "deprecated"
version = "1.2.18"
description = "Python @deprecated decorator to deprecate old python classes, functions or methods."
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7"
files = [
{file = "Deprecated-1.2.18-py2.py3-none-any.whl", hash = "sha256:bd5011788200372a32418f888e326a09ff80d0214bd961147cfed01b5c018eec"},
{file = "deprecated-1.2.18.tar.gz", hash = "sha256:422b6f6d859da6f2ef57857761bfb392480502a64c3028ca9bbe86085d72115d"},
]
[package.dependencies]
wrapt = ">=1.10,<2"
[package.extras]
dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "setuptools", "tox"]
[[package]]
name = "dj-database-url"
version = "0.5.0"
@ -2270,6 +2305,17 @@ files = [
{file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
]
[[package]]
name = "ndjson"
version = "0.3.1"
description = "JsonDecoder for ndjson"
optional = false
python-versions = "*"
files = [
{file = "ndjson-0.3.1-py2.py3-none-any.whl", hash = "sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410"},
{file = "ndjson-0.3.1.tar.gz", hash = "sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6"},
]
[[package]]
name = "oauthlib"
version = "3.2.2"
@ -3792,20 +3838,6 @@ files = [
[package.dependencies]
types-urllib3 = "*"
[[package]]
name = "types-requests"
version = "2.32.0.20241016"
description = "Typing stubs for requests"
optional = false
python-versions = ">=3.8"
files = [
{file = "types-requests-2.32.0.20241016.tar.gz", hash = "sha256:0d9cad2f27515d0e3e3da7134a1b6f28fb97129d86b867f24d9c726452634d95"},
{file = "types_requests-2.32.0.20241016-py3-none-any.whl", hash = "sha256:4195d62d6d3e043a4eaaf08ff8a62184584d2e8684e9d2aa178c7915a7da3747"},
]
[package.dependencies]
urllib3 = ">=2"
[[package]]
name = "types-urllib3"
version = "1.26.25.14"
@ -3872,23 +3904,6 @@ brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotl
secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"]
socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"]
[[package]]
name = "urllib3"
version = "2.3.0"
description = "HTTP library with thread-safe connection pooling, file post, and more."
optional = false
python-versions = ">=3.9"
files = [
{file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"},
{file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"},
]
[package.extras]
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
h2 = ["h2 (>=4,<5)"]
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
zstd = ["zstandard (>=0.18.0)"]
[[package]]
name = "vine"
version = "5.1.0"
@ -4168,4 +4183,4 @@ type = ["pytest-mypy"]
[metadata]
lock-version = "2.0"
python-versions = ">=3.9,<4.0"
content-hash = "47922140929eccdcdf8eabb43e4b34af9ce0c5d395948385e7a861036c43ab3c"
content-hash = "1b3b34d6e5e5db0f5192c5d2f3054cf2b747f2833a4304167d55c6e1c8e0de00"

2
pyproject.toml
View File

@ -47,8 +47,10 @@ thefuzz = "^0.22.1"
dataclass-wizard = "0.22.0"
webdavclient3 = "^3.14.6"
boto3 = "^1.35.37"
urllib3 = "<2"
django-oauth-toolkit = "^3.0.1"
meta-yt = "^0.1.9"
berserk = "^0.13.2"
[tool.poetry.group.dev]
optional = true

1
vrobbler.conf.example
View File

@ -24,6 +24,7 @@ VROBBLER_COMICVINE_API_KEY="<key>"
VROBBLER_TODOIST_CLIENT_ID="<id>"
VROBBLER_TODOIST_CLIENT_SECRET="<key>"
VROBBLER_GOOGLE_API_KEY="<key>"
VROBBLER_LICHESS_API_KEY = "<key>"
# Storages
# VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"

119
vrobbler/apps/boardgames/sources/lichess.py Normal file
View File

@ -0,0 +1,119 @@
import berserk
from django.conf import settings
from boardgames.models import BoardGame
from scrobbles.models import Scrobble
from django.contrib.auth import get_user_model
User = get_user_model()
def import_chess_games_for_all_users():
client = berserk.Client(
session=berserk.TokenSession(settings.LICHESS_API_KEY)
)
scrobbles_to_create = []
for user in User.objects.filter(profile__lichess_username__isnull=False):
games = client.games.export_by_player(user.profile.lichess_username)
for game_dict in games:
chess, created = BoardGame.objects.get_or_create(title="Chess")
if created:
chess.run_time_seconds = 1800
chess.bggeek_id = 171
chess.save(update_fields=["run_time_seconds", "bggeek_id"])
scrobble = Scrobble.objects.filter(
user_id=user.id,
timestamp=game_dict.get("createdAt"),
board_game_id=chess.id,
).first()
if scrobble:
continue
log_data = {
"variant": game_dict.get("variant"),
"lichess_id": game_dict.get("id"),
"rated": game_dict.get("rated"),
"speed": game_dict.get("speed"),
"moves": game_dict.get("moves"),
"players": [],
}
chess_status = game_dict.get("status")
chess_source = game_dict.get("source")
winner = game_dict.get("winner")
black_player = game_dict.get("players", {}).get("black", {})
white_player = game_dict.get("players", {}).get("white", {})
user_player = {
"user_id": user.profile.lichess_username,
"color": "",
"win": False,
}
other_player = {"name_str": "", "color": "", "win": False}
if (
black_player.get("user", {}).get("name", "")
== user.profile.lichess_username
):
user_player["color"] = "black"
if "aiLevel" in white_player.keys():
other_player["name_str"] = "aiLevel_" + str(
white_player.get("aiLevel", "")
)
else:
other_player["name_str"] = white_player.get(
"user", {}
).get("name", "")
other_player["color"] = "white"
if winner == "black":
user_player["win"] = True
else:
other_player["win"] = True
if (
white_player.get("user", {}).get("name", "")
== user.profile.lichess_username
):
user_player["color"] = "white"
if "aiLevel" in black_player.keys():
other_player["name_str"] = "aiLevel_" + str(
black_player.get("aiLevel", "")
)
else:
other_player["name_str"] = white_player.get(
"user", {}
).get("name", "")
other_player["color"] = "black"
if winner == "white":
user_player["win"] = True
else:
other_player["win"] = True
log_data["players"].append(user_player)
log_data["players"].append(other_player)
scrobble_dict = {
"user_id": user.id,
"timestamp": game_dict.get("createdAt"),
"stop_timestamp": game_dict.get("lastMoveAt"),
"board_game_id": chess.id,
"log": log_data,
}
scrobbles_to_create.append(Scrobble(**scrobble_dict))
if scrobbles_to_create:
Scrobble.objects.bulk_create(scrobbles_to_create)
return scrobbles_to_create
# 'players': {
# 'white': {'aiLevel': 1},
# 'black': {'user': {'name': 'secstate', 'id': 'secstate'},
# 'rating': 1500,
# 'provisional': True}
# },
# 'fullId': '4T8CinfXdI95',
# 'winner': 'black',

21
vrobbler/apps/profiles/migrations/0021_userprofile_lichess_username.py Normal file
View File

@ -0,0 +1,21 @@
# Generated by Django 4.2.18 on 2025-01-29 04:50
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
(
"profiles",
"0020_userprofile_ntfy_enabled_userprofile_ntfy_url_and_more",
),
]
operations = [
migrations.AddField(
model_name="userprofile",
name="lichess_username",
field=models.CharField(blank=True, max_length=255, null=True),
),
]

1
vrobbler/apps/profiles/models.py
View File

@ -30,6 +30,7 @@ class UserProfile(TimeStampedModel):
archivebox_url = models.CharField(max_length=255, **BNULL)
bgg_username = models.CharField(max_length=255, **BNULL)
lichess_username = models.CharField(max_length=255, **BNULL)
todoist_auth_key = EncryptedField(**BNULL)
todoist_state = EncryptedField(**BNULL)

10
vrobbler/apps/scrobbles/management/commands/import_from_lichess.py Normal file
View File

@ -0,0 +1,10 @@
from django.core.management.base import BaseCommand
from vrobbler.apps.boardgames.sources.lichess import (
import_chess_games_for_all_users,
)
class Command(BaseCommand):
def handle(self, *args, **options):
count = len(import_chess_games_for_all_users())
print(f"Imported {count} Lichess games")

6
vrobbler/settings-testing.py
View File

@ -58,6 +58,12 @@ LASTFM_SECRET_KEY = os.getenv("VROBBLER_LASTFM_SECRET_KEY")
IGDB_CLIENT_ID = os.getenv("VROBBLER_IGDB_CLIENT_ID")
IGDB_CLIENT_SECRET = os.getenv("VROBBLER_IGDB_CLIENT_SECRET")
TODOIST_CLIENT_ID = os.getenv("VROBBLER_TODOIST_CLIENT_ID", "")
TODOIST_CLIENT_SECRET = os.getenv("VROBBLER_TODOIST_CLIENT_SECRET", "")
GOOGLE_API_KEY = os.getenv("VROBBLER_GOOGLE_API_KEY", "")
LICHESS_API_KEY = os.getenv("VROBBLER_LICHESS_API_KEY", "")
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
TIME_ZONE = os.getenv("VROBBLER_TIME_ZONE", "US/Eastern")

1
vrobbler/settings.py
View File

@ -75,6 +75,7 @@ TODOIST_CLIENT_ID = os.getenv("VROBBLER_TODOIST_CLIENT_ID", "")
TODOIST_CLIENT_SECRET = os.getenv("VROBBLER_TODOIST_CLIENT_SECRET", "")
GOOGLE_API_KEY = os.getenv("VROBBLER_GOOGLE_API_KEY", "")
LICHESS_API_KEY = os.getenv("VROBBLER_LICHESS_API_KEY", "")
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"