secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Boom. Video game metadata

Browse Source
This commit is contained in:
Colin Powell
2023-03-05 16:36:36 -05:00
parent a0e852775c
commit c757e743ac
15 changed files with 1006 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
vrobbler/apps/music/models.py
View File

@ -1,3 +1,4 @@
import requests
import logging
from tempfile import NamedTemporaryFile
from typing import Dict, Optional
@ -70,11 +71,10 @@ class Artist(TimeStampedModel):
self.theaudiodb_genre = tadb_info["genre"]
self.theaudiodb_mood = tadb_info["mood"]
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urlopen(tadb_info["thumb_url"]).read())
img_temp.flush()
img_filename = f"{self.name}_{self.uuid}.jpg"
self.thumbnail.save(img_filename, File(img_temp))
r = requests.get(tadb_info.get("thumb_url", ""))
if r.status_code == 200:
fname = f"{self.name}_{self.uuid}.jpg"
self.thumbnail.save(fname, ContentFile(r.content), save=True)
@property
def rym_link(self):

29
vrobbler/apps/profiles/migrations/0005_remove_userprofile_twitch_client_id_and_more.py Normal file
View File

@ -0,0 +1,29 @@
# Generated by Django 4.1.5 on 2023-03-05 21:27
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("profiles", "0004_userprofile_twitch_token_and_more"),
]
operations = [
migrations.RemoveField(
model_name="userprofile",
name="twitch_client_id",
),
migrations.RemoveField(
model_name="userprofile",
name="twitch_client_secret",
),
migrations.RemoveField(
model_name="userprofile",
name="twitch_token",
),
migrations.RemoveField(
model_name="userprofile",
name="twitch_token_expires",
),
]

17
vrobbler/apps/profiles/models.py
View File

@ -8,8 +8,6 @@ from django_extensions.db.models import TimeStampedModel
from encrypted_field import EncryptedField
from profiles.constants import PRETTY_TIMEZONE_CHOICES
from vrobbler.apps.videogames.igdb import refresh_igdb_api_token
User = get_user_model()
BNULL = {"blank": True, "null": True}
@ -25,10 +23,6 @@ class UserProfile(TimeStampedModel):
)
lastfm_username = models.CharField(max_length=255, **BNULL)
lastfm_password = EncryptedField(**BNULL)
twitch_client_id = models.CharField(max_length=255, **BNULL)
twitch_client_secret = EncryptedField(**BNULL)
twitch_token = EncryptedField(**BNULL)
twitch_token_expires = models.DateTimeField(**BNULL)
def __str__(self):
return f"User profile for {self.user}"
@ -36,14 +30,3 @@ class UserProfile(TimeStampedModel):
@property
def tzinfo(self):
return pytz.timezone(self.timezone)
def get_twitch_token(self):
now = timezone.now()
token = self.twitch_token
if not token or self.twitch_token_expires < now:
self.twitch_token, expires_in = refresh_igdb_api_token(
self.user_id
)
self.twitch_token_expires = now + timedelta(seconds=expires_in)
self.save(update_fields=["twitch_token", "twitch_token_expires"])
return self.twitch_token

12
vrobbler/apps/scrobbles/mixins.py
View File

@ -1,3 +1,5 @@
import logging
from typing import Dict
from uuid import uuid4
from django.db import models
@ -5,6 +7,8 @@ from django_extensions.db.models import TimeStampedModel
BNULL = {"blank": True, "null": True}
logger = logging.getLogger(__name__)
class ScrobblableMixin(TimeStampedModel):
SECONDS_TO_STALE = 1600
@ -13,7 +17,13 @@ class ScrobblableMixin(TimeStampedModel):
title = models.CharField(max_length=255, **BNULL)
run_time = models.CharField(max_length=8, **BNULL)
run_time_ticks = models.PositiveBigIntegerField(**BNULL)
# thumbs = models.IntegerField(default=Opinion.NEUTRAL, choices=Opinion.choices)
class Meta:
abstract = True
def fix_metadata(self):
logger.warn("fix_metadata() not implemented yet")
@classmethod
def find_or_create(cls):
logger.warn("find_or_create() not implemented yet")

8
vrobbler/apps/videogames/admin.py
View File

@ -19,7 +19,13 @@ class VideoGameCollectionAdmin(admin.ModelAdmin):
@admin.register(VideoGame)
class GameAdmin(admin.ModelAdmin):
date_hierarchy = "created"
list_display = ("title", "igdb_id")
list_display = (
"hltb_id",
"title",
"hltb_score",
"main_story_time",
"release_year",
)
ordering = ("-created",)
inlines = [
ScrobbleInline,

52
vrobbler/apps/videogames/howlongtobeat.py Normal file
View File

@ -0,0 +1,52 @@
import logging
from typing import Optional
from howlongtobeatpy import HowLongToBeat
logger = logging.getLogger(__name__)
def lookup_game_from_hltb(name_or_id: str) -> Optional[dict]:
"""Lookup game on HowLongToBeat.com via HLtB ID or a name string and return
the data in a dictonary mapped to our internal game fields
"""
hltb_game = {}
try:
hltb_id = int(name_or_id)
except ValueError:
hltb_id = None
if hltb_id:
hltb_game = HowLongToBeat().search_from_id(hltb_id)
logger.info(f"Found game on HLtB for ID {hltb_id}")
if not hltb_game:
results = HowLongToBeat().search(name_or_id)
if not results:
logger.warn(f"Lookup of game on HLtB failed for ID {name_or_id}")
return
hltb_game = results[0]
if len(results) > 1:
found_games = []
for g in results:
found_games.append(f"{g.game_name} ({g.game_id})")
logger.info(
f"Found more than one match {found_games}, taking {hltb_game}"
)
game_dict = {
"title": hltb_game.game_name,
"hltb_id": hltb_game.game_id,
"main_story_time": hrs_to_secs(hltb_game.main_story),
"main_extra_time": hrs_to_secs(hltb_game.main_extra),
"completionist_time": hrs_to_secs(hltb_game.completionist),
"release_year": hltb_game.release_world,
"hltb_score": hltb_game.review_score,
"platforms": hltb_game.profile_platforms,
"cover_url": hltb_game.game_image_url,
}
return game_dict

97
vrobbler/apps/videogames/igdb.py
View File

@ -3,6 +3,7 @@ import logging
from datetime import datetime
from typing import Dict, Tuple
from django.conf import settings
import pytz
import requests
from django.contrib.auth import get_user_model
@ -18,68 +19,74 @@ ALT_NAMES_URL = "https://api.igdb.com/v4/alternative_names"
SCREENSHOT_URL = "https://api.igdb.com/v4/screenshots"
COVER_URL = "https://api.igdb.com/v4/covers"
IGDB_CLIENT_ID = getattr(settings, "IGDB_CLIENT_ID")
IGDB_CLIENT_SECRET = getattr(settings, "IGDB_CLIENT_SECRET")
logger = logging.getLogger(__name__)
User = get_user_model()
def refresh_igdb_api_token(user_id: int) -> Tuple[str, int]:
user = User.objects.get(id=user_id)
def get_igdb_token() -> str:
token_url = REFRESH_TOKEN_URL.format(
id=user.profile.twitch_client_id,
secret=user.profile.twitch_client_secret,
id=IGDB_CLIENT_ID, secret=IGDB_CLIENT_SECRET
)
response = requests.post(token_url)
results = json.loads(response.content)
return results.get("access_token"), results.get("expires_in")
return results.get("access_token")
def lookup_game_from_igdb(client_id: str, token: str, game_id: str) -> Dict:
def lookup_game_from_igdb(igdb_id: str) -> Dict:
"""Given credsa and an IGDB game ID, lookup the game metadata and return it
in a dictionary mapped to our internal game fields
"""
headers = {
"Authorization": f"Bearer {token}",
"Client-ID": client_id,
"Authorization": f"Bearer {get_igdb_token()}",
"Client-ID": IGDB_CLIENT_ID,
}
fields = "id,name,alternative_names.*,release_dates.*,cover.*,screenshots.*,rating,rating_count"
game_dict = {}
if game_id:
body = f"fields {fields}; where id = {game_id};"
response = requests.post(GAMES_URL, data=body, headers=headers)
results = json.loads(response.content)
if results:
game = results[0]
logger.debug(game)
body = f"fields {fields}; where id = {igdb_id};"
response = requests.post(GAMES_URL, data=body, headers=headers)
results = json.loads(response.content)
if not results:
logger.warn(f"Lookup of game on IGDB failed for ID {igdb_id}")
return game_dict
alt_name = None
if "alternative_names" in game.keys():
alt_name = game.get("alternative_names")[0].get("name")
screenshot_url = None
if "screenshots" in game.keys():
screenshot_url = "https:" + game.get("screenshots")[0].get(
"url"
).replace("t_thumb", "t_screenshot_big_2x")
cover_url = None
if "cover" in game.keys():
cover_url = "https:" + game.get("cover").get("url").replace(
"t_thumb", "t_cover_big_2x"
)
release_date = None
if "release_dates" in game.keys():
release_date = game.get("release_dates")[0].get("date")
if release_date:
release_date = datetime.utcfromtimestamp(
release_date
).replace(tzinfo=pytz.utc)
game = results[0]
game_dict = {
"igdb_id": game.get("id"),
"title": game.get("name"),
"alternative_name": alt_name,
"screenshot_url": screenshot_url,
"cover_url": cover_url,
"rating": game.get("rating"),
"rating_count": game.get("rating_count"),
"release_date": release_date,
}
alt_name = None
if "alternative_names" in game.keys():
alt_name = game.get("alternative_names")[0].get("name")
screenshot_url = None
if "screenshots" in game.keys():
screenshot_url = "https:" + game.get("screenshots")[0].get(
"url"
).replace("t_thumb", "t_screenshot_big_2x")
cover_url = None
if "cover" in game.keys():
cover_url = "https:" + game.get("cover").get("url").replace(
"t_thumb", "t_cover_big_2x"
)
release_date = None
if "release_dates" in game.keys():
release_date = game.get("release_dates")[0].get("date")
if release_date:
release_date = datetime.utcfromtimestamp(release_date).replace(
tzinfo=pytz.utc
)
game_dict = {
"igdb_id": game.get("id"),
"title": game.get("name"),
"alternative_name": alt_name,
"screenshot_url": screenshot_url,
"cover_url": cover_url,
"rating": game.get("rating"),
"rating_count": game.get("rating_count"),
"release_date": release_date,
}
return game_dict

119
vrobbler/apps/videogames/migrations/0003_videogameplatform_videogame_completionist_time_and_more.py Normal file
View File

@ -0,0 +1,119 @@
# Generated by Django 4.1.5 on 2023-03-05 18:58
from django.db import migrations, models
import django.db.models.deletion
import django_extensions.db.fields
import uuid
class Migration(migrations.Migration):
dependencies = [
("videogames", "0002_videogame_release_date"),
]
operations = [
migrations.CreateModel(
name="VideoGamePlatform",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
(
"created",
django_extensions.db.fields.CreationDateTimeField(
auto_now_add=True, verbose_name="created"
),
),
(
"modified",
django_extensions.db.fields.ModificationDateTimeField(
auto_now=True, verbose_name="modified"
),
),
("name", models.CharField(max_length=255)),
(
"uuid",
models.UUIDField(
blank=True,
default=uuid.uuid4,
editable=False,
null=True,
),
),
(
"logo",
models.ImageField(
blank=True,
null=True,
upload_to="games/platform-logos/",
),
),
("igdb_id", models.IntegerField(blank=True, null=True)),
],
options={
"get_latest_by": "modified",
"abstract": False,
},
),
migrations.AddField(
model_name="videogame",
name="completionist_time",
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name="videogame",
name="hltb_cover",
field=models.ImageField(
blank=True, null=True, upload_to="games/hltb_covers/"
),
),
migrations.AddField(
model_name="videogame",
name="hltb_id",
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name="videogame",
name="hltb_score",
field=models.FloatField(blank=True, null=True),
),
migrations.AddField(
model_name="videogame",
name="main_extra_time",
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name="videogame",
name="main_story_time",
field=models.IntegerField(blank=True, null=True),
),
migrations.AddField(
model_name="videogame",
name="release_year",
field=models.IntegerField(blank=True, null=True),
),
migrations.AlterField(
model_name="videogame",
name="screenshot",
field=models.ImageField(
blank=True, null=True, upload_to="games/screenshots/"
),
),
migrations.AddField(
model_name="videogame",
name="platform",
field=models.ForeignKey(
blank=True,
null=True,
on_delete=django.db.models.deletion.DO_NOTHING,
to="videogames.videogameplatform",
),
),
]

32
vrobbler/apps/videogames/migrations/0004_alter_videogame_alternative_name_and_more.py Normal file
View File

@ -0,0 +1,32 @@
# Generated by Django 4.1.5 on 2023-03-05 19:58
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
(
"videogames",
"0003_videogameplatform_videogame_completionist_time_and_more",
),
]
operations = [
migrations.AlterField(
model_name="videogame",
name="alternative_name",
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.RemoveField(
model_name="videogame",
name="platform",
),
migrations.AddField(
model_name="videogame",
name="platform",
field=models.ManyToManyField(
blank=True, null=True, to="videogames.videogameplatform"
),
),
]

18
vrobbler/apps/videogames/migrations/0005_rename_platform_videogame_platforms.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 4.1.5 on 2023-03-05 19:59
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("videogames", "0004_alter_videogame_alternative_name_and_more"),
]
operations = [
migrations.RenameField(
model_name="videogame",
old_name="platform",
new_name="platforms",
),
]

70
vrobbler/apps/videogames/models.py
View File

@ -1,5 +1,5 @@
import logging
from typing import Dict
from typing import Optional
from uuid import uuid4
from django.conf import settings
@ -8,10 +8,26 @@ from django.urls import reverse
from django_extensions.db.models import TimeStampedModel
from scrobbles.mixins import ScrobblableMixin
logger = logging.getLogger(__name__)
BNULL = {"blank": True, "null": True}
class VideoGamePlatform(TimeStampedModel):
name = models.CharField(max_length=255)
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
logo = models.ImageField(upload_to="games/platform-logos/", **BNULL)
igdb_id = models.IntegerField(**BNULL)
def __str__(self):
return self.name
def get_absolute_url(self):
return reverse(
"videogames:platform_detail", kwargs={"slug": self.uuid}
)
class VideoGameCollection(TimeStampedModel):
name = models.CharField(max_length=255)
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
@ -23,22 +39,48 @@ class VideoGameCollection(TimeStampedModel):
def get_absolute_url(self):
return reverse(
"videogames:videogamecollection_detail", kwargs={"slug": self.uuid}
"videogames:collection_detail", kwargs={"slug": self.uuid}
)
class VideoGame(ScrobblableMixin):
COMPLETION_PERCENT = getattr(settings, "GAME_COMPLETION_PERCENT", 100)
FIELDS_FROM_IGDB = [
"igdb_id",
"alternative_name",
"rating",
"rating_count",
"release_date",
"cover",
"screenshot",
]
FIELDS_FROM_HLTB = [
"hltb_id",
"release_year",
"main_story_time",
"main_extra_time",
"completionist_time",
"hltb_score",
]
title = models.CharField(max_length=255)
igdb_id = models.IntegerField(**BNULL)
alternative_name = models.CharField(max_length=255)
hltb_id = models.IntegerField(**BNULL)
alternative_name = models.CharField(max_length=255, **BNULL)
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
cover = models.ImageField(upload_to="games/covers/", **BNULL)
screenshot = models.ImageField(upload_to="games/covers/", **BNULL)
screenshot = models.ImageField(upload_to="games/screenshots/", **BNULL)
hltb_cover = models.ImageField(upload_to="games/hltb_covers/", **BNULL)
rating = models.FloatField(**BNULL)
rating_count = models.IntegerField(**BNULL)
release_date = models.DateTimeField(**BNULL)
release_year = models.IntegerField(**BNULL)
main_story_time = models.IntegerField(**BNULL)
main_extra_time = models.IntegerField(**BNULL)
completionist_time = models.IntegerField(**BNULL)
hltb_score = models.FloatField(**BNULL)
platforms = models.ManyToManyField(VideoGamePlatform)
def __str__(self):
return self.title
@ -48,6 +90,20 @@ class VideoGame(ScrobblableMixin):
"videogames:videogame_detail", kwargs={"slug": self.uuid}
)
@classmethod
def find_or_create(cls, data_dict: Dict) -> "VideoGame":
...
def hltb_link(self):
return f"https://howlongtobeat.com/game/{self.hltb_id}"
def fix_metadata(
self,
force_update: bool = False,
):
from videogames.utils import (
get_or_create_videogame,
load_game_data_from_igdb,
)
if self.hltb_id and force_update:
get_or_create_videogame(str(self.hltb_id), force_update)
if self.igdb_id:
load_game_data_from_igdb(self.id)

110
vrobbler/apps/videogames/utils.py
View File

@ -1,42 +1,96 @@
import logging
from tempfile import NamedTemporaryFile
from urllib.request import urlopen
from typing import Optional
from django.core.files.base import File
import requests
from django.core.files.base import ContentFile
from videogames.models import VideoGame, VideoGamePlatform
from videogames.models import VideoGame
from vrobbler.apps.videogames.igdb import lookup_game_from_igdb
from videogames.howlongtobeat import lookup_game_from_hltb
from videogames.igdb import lookup_game_from_igdb
logger = logging.getLogger(__name__)
def get_or_create_videogame(
client_id: str, token: str, igdb_id: str
) -> VideoGame:
game = None
logger.debug(f"Looking up video game {igdb_id}")
game_dict = lookup_game_from_igdb(client_id, token, igdb_id)
def hrs_to_secs(hrs: float) -> int:
return int(hrs * 60 * 60)
game = VideoGame.objects.filter(igdb_id=igdb_id).first()
if not game:
screenshot_url = game_dict.pop("screenshot_url")
def get_or_create_videogame(
name_or_id: str, force_update=False
) -> Optional[VideoGame]:
"""Look up game by name or ID from HowLongToBeat"""
game_dict = lookup_game_from_hltb(name_or_id)
if not game_dict:
return
platform_ids = []
for platform in game_dict.get("platforms"):
p, _created = VideoGamePlatform.objects.get_or_create(name=platform)
platform_ids.append(p.id)
game_dict.pop("platforms")
game, game_created = VideoGame.objects.get_or_create(
hltb_id=game_dict.get("hltb_id")
)
if game_created or force_update:
cover_url = game_dict.pop("cover_url")
game = VideoGame.objects.create(**game_dict)
VideoGame.objects.filter(pk=game.id).update(**game_dict)
game.refresh_from_db()
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urlopen(screenshot_url).read())
img_temp.flush()
img_filename = f"{game.title}_{game.uuid}.jpg"
game.screenshot.save(img_filename, File(img_temp))
# Associate plaforms
if platform_ids:
game.platforms.add(*platform_ids)
img_temp = NamedTemporaryFile(delete=True)
img_temp.write(urlopen(cover_url).read())
img_temp.flush()
img_filename = f"{game.title}_{game.uuid}.jpg"
game.cover.save(img_filename, File(img_temp))
logger.debug(f"Created video game {game.title} ({game.igdb_id}) ")
# Go get cover image if the URL is present
if cover_url and not game.hltb_cover:
headers = {"User-Agent": "Vrobbler 0.11.12"}
r = requests.get(cover_url, headers=headers)
logger.debug(r.status_code)
if r.status_code == 200:
fname = f"{game.title}_cover_{game.uuid}.jpg"
game.hltb_cover.save(fname, ContentFile(r.content), save=True)
logger.debug("Loaded cover image from HLtB")
return game
def load_game_data_from_igdb(game_id: int) -> Optional[VideoGame]:
"""Look up game, if it doesn't exist, lookup data from igdb"""
game = VideoGame.objects.filter(id=game_id).first()
if not game:
logger.warn(f"Video game with ID {game_id} not found")
return
if not game.igdb_id:
logger.warn(f"Video game has no IGDB ID")
return
logger.info(f"Looking up video game {game.igdb_id}")
game_dict = lookup_game_from_igdb(game.igdb_id)
if not game_dict:
logger.warn(f"No game data found on IGDB for ID {game.igdb_id}")
return
screenshot_url = game_dict.pop("screenshot_url")
cover_url = game_dict.pop("cover_url")
VideoGame.objects.filter(pk=game.id).update(**game_dict)
game.refresh_from_db()
if not game.screenshot:
r = requests.get(screenshot_url)
if r.status_code == 200:
fname = f"{game.title}_{game.uuid}.jpg"
game.screenshot.save(fname, ContentFile(r.content), save=True)
if not game.cover:
r = requests.get(cover_url)
if r.status_code == 200:
fname = f"{game.title}_{game.uuid}.jpg"
game.cover.save(fname, ContentFile(r.content), save=True)
return game

2
vrobbler/settings.py
View File

@ -55,6 +55,8 @@ THEAUDIODB_API_KEY = os.getenv("VROBBLER_THEAUDIODB_API_KEY", "2")
TMDB_API_KEY = os.getenv("VROBBLER_TMDB_API_KEY", "")
LASTFM_API_KEY = os.getenv("VROBBLER_LASTFM_API_KEY")
LASTFM_SECRET_KEY = os.getenv("VROBBLER_LASTFM_SECRET_KEY")
IGDB_CLIENT_ID = os.getenv("VROBBLER_IGDB_CLIENT_ID")
IGDB_CLIENT_SECRET = os.getenv("VROBBLER_IGDB_CLIENT_SECRET")
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"