Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 68ff230f13 | |||
| 57a952a6d1 | |||
| 718fcf7392 |
10
PROJECT.org
10
PROJECT.org
@ -591,6 +591,16 @@ a helper method to create board game scrobbles given a json blob. It's
|
|||||||
independent of the email flow it was originally creatdd for
|
independent of the email flow it was originally creatdd for
|
||||||
|
|
||||||
** TODO [#B] Is there way to create unique slugs for media instances :media_types:
|
** TODO [#B] Is there way to create unique slugs for media instances :media_types:
|
||||||
|
* Version 55.2 [2/2]
|
||||||
|
** DONE [#A] Fix bug in scrobble id in calendar view :templates:
|
||||||
|
:PROPERTIES:
|
||||||
|
:ID: 8cb34852-b18f-e794-cd9b-fb1ecad70a0d
|
||||||
|
:END:
|
||||||
|
** DONE [#A] Video game cleanup script should clear out broken images :metadata:videogames:
|
||||||
|
:PROPERTIES:
|
||||||
|
:ID: ca1f1ea9-0f79-082c-5ff7-867671faff4b
|
||||||
|
:END:
|
||||||
|
|
||||||
* Version 55.1 [1/1]
|
* Version 55.1 [1/1]
|
||||||
** DONE [#A] Clean up metadata scrapping for video games :metadata:videogames:
|
** DONE [#A] Clean up metadata scrapping for video games :metadata:videogames:
|
||||||
:PROPERTIES:
|
:PROPERTIES:
|
||||||
|
|||||||
@ -1,6 +1,6 @@
|
|||||||
[tool.poetry]
|
[tool.poetry]
|
||||||
name = "vrobbler"
|
name = "vrobbler"
|
||||||
version = "55.1"
|
version = "55.2"
|
||||||
description = ""
|
description = ""
|
||||||
authors = ["Colin Powell <colin@unbl.ink>"]
|
authors = ["Colin Powell <colin@unbl.ink>"]
|
||||||
|
|
||||||
|
|||||||
@ -1717,6 +1717,7 @@ class ScrobbleCalendarView(LoginRequiredMixin, TemplateView):
|
|||||||
for scrobble in day_map[day_num]:
|
for scrobble in day_map[day_num]:
|
||||||
day_scrobbles.append(
|
day_scrobbles.append(
|
||||||
{
|
{
|
||||||
|
"id": scrobble.pk,
|
||||||
"uuid": scrobble.uuid,
|
"uuid": scrobble.uuid,
|
||||||
"emoji": self.MEDIA_EMOJI.get(scrobble.media_type, "📌"),
|
"emoji": self.MEDIA_EMOJI.get(scrobble.media_type, "📌"),
|
||||||
"title": (
|
"title": (
|
||||||
|
|||||||
@ -69,6 +69,11 @@ class Command(BaseCommand):
|
|||||||
type=int,
|
type=int,
|
||||||
help="Only process a specific game by ID",
|
help="Only process a specific game by ID",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fix-broken-images",
|
||||||
|
action="store_true",
|
||||||
|
help="Check and refetch broken/deleted game images (cover, screenshot, hltb_cover)",
|
||||||
|
)
|
||||||
for flag in MISSING_ALL:
|
for flag in MISSING_ALL:
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
f"--missing-{flag}",
|
f"--missing-{flag}",
|
||||||
@ -92,15 +97,16 @@ class Command(BaseCommand):
|
|||||||
sleep_secs = options["sleep"]
|
sleep_secs = options["sleep"]
|
||||||
force = options["force"]
|
force = options["force"]
|
||||||
game_id = options["game_id"]
|
game_id = options["game_id"]
|
||||||
|
fix_broken_images = options.get("fix_broken_images", False)
|
||||||
flags = options.get("missing_flags") or []
|
flags = options.get("missing_flags") or []
|
||||||
all_missing = options["all_missing"]
|
all_missing = options["all_missing"]
|
||||||
|
|
||||||
if all_missing:
|
if all_missing:
|
||||||
flags = MISSING_ALL
|
flags = MISSING_ALL
|
||||||
|
|
||||||
if not flags and not game_id and not force:
|
if not flags and not game_id and not force and not fix_broken_images:
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
"No filters specified. Use --all, --missing-*, --game-id, or --force."
|
"No filters specified. Use --all, --missing-*, --game-id, --force, or --fix-broken-images."
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
@ -135,25 +141,33 @@ class Command(BaseCommand):
|
|||||||
"release_year_fixed": 0,
|
"release_year_fixed": 0,
|
||||||
"igdb_id_found": 0,
|
"igdb_id_found": 0,
|
||||||
"hltb_id_found": 0,
|
"hltb_id_found": 0,
|
||||||
|
"images_fixed": 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
for batch_num, offset in enumerate(range(0, len(qs), batch_size)):
|
enriched_any = bool(flags or game_id or force)
|
||||||
batch = qs[offset : offset + batch_size]
|
|
||||||
for game in batch:
|
|
||||||
result = self._enrich_game(game, sleep_secs, force)
|
|
||||||
self._check_retroarch_name(game, title_mismatches)
|
|
||||||
if result:
|
|
||||||
enriched += 1
|
|
||||||
for key in stats:
|
|
||||||
if result.get(key):
|
|
||||||
stats[key] += 1
|
|
||||||
else:
|
|
||||||
skipped += 1
|
|
||||||
|
|
||||||
self.stdout.write(
|
if enriched_any:
|
||||||
f" Batch {batch_num + 1}: {offset + len(batch)}/{total} — "
|
for batch_num, offset in enumerate(range(0, len(qs), batch_size)):
|
||||||
f"enriched: {enriched}, skipped: {skipped}"
|
batch = qs[offset : offset + batch_size]
|
||||||
)
|
for game in batch:
|
||||||
|
result = self._enrich_game(game, sleep_secs, force)
|
||||||
|
self._check_retroarch_name(game, title_mismatches)
|
||||||
|
if result:
|
||||||
|
enriched += 1
|
||||||
|
for key in stats:
|
||||||
|
if result.get(key):
|
||||||
|
stats[key] += 1
|
||||||
|
else:
|
||||||
|
skipped += 1
|
||||||
|
|
||||||
|
self.stdout.write(
|
||||||
|
f" Batch {batch_num + 1}: {offset + len(batch)}/{total} — "
|
||||||
|
f"enriched: {enriched}, skipped: {skipped}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if fix_broken_images:
|
||||||
|
broken_stats = self._fix_broken_images(qs, sleep_secs)
|
||||||
|
stats["images_fixed"] = broken_stats["images_fixed"]
|
||||||
|
|
||||||
self.stdout.write(
|
self.stdout.write(
|
||||||
f"\nResults (commit={commit}):\n"
|
f"\nResults (commit={commit}):\n"
|
||||||
@ -168,6 +182,8 @@ class Command(BaseCommand):
|
|||||||
f" IGDB IDs found: {stats['igdb_id_found']}\n"
|
f" IGDB IDs found: {stats['igdb_id_found']}\n"
|
||||||
f" HLtB IDs found: {stats['hltb_id_found']}"
|
f" HLtB IDs found: {stats['hltb_id_found']}"
|
||||||
)
|
)
|
||||||
|
if fix_broken_images:
|
||||||
|
self.stdout.write(f" Broken images fixed: {stats['images_fixed']}")
|
||||||
|
|
||||||
if title_mismatches:
|
if title_mismatches:
|
||||||
self.stdout.write("\nTitle vs retroarch_name mismatches (not auto-fixed):")
|
self.stdout.write("\nTitle vs retroarch_name mismatches (not auto-fixed):")
|
||||||
@ -450,3 +466,64 @@ class Command(BaseCommand):
|
|||||||
self.stdout.write(f" [TAG] {game} — tagged hltb-enriched")
|
self.stdout.write(f" [TAG] {game} — tagged hltb-enriched")
|
||||||
|
|
||||||
return changed if any(changed.values()) else None
|
return changed if any(changed.values()) else None
|
||||||
|
|
||||||
|
def _fix_broken_images(self, games, sleep_secs):
|
||||||
|
from django.core.files.base import ContentFile
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from videogames.igdb import lookup_game_from_igdb
|
||||||
|
from videogames.howlongtobeat import lookup_game_from_hltb
|
||||||
|
|
||||||
|
stats = {"cover_fixed": 0, "screenshot_fixed": 0, "images_fixed": 0}
|
||||||
|
|
||||||
|
for game in games:
|
||||||
|
for field_name, source in [
|
||||||
|
("cover", "igdb"),
|
||||||
|
("screenshot", "igdb"),
|
||||||
|
("hltb_cover", "hltb"),
|
||||||
|
]:
|
||||||
|
field = getattr(game, field_name)
|
||||||
|
if not field.name:
|
||||||
|
continue
|
||||||
|
if field.storage.exists(field.name):
|
||||||
|
continue
|
||||||
|
|
||||||
|
self.stdout.write(
|
||||||
|
f" [IMAGE] {game} — {field_name} is broken (file missing), refetching…"
|
||||||
|
)
|
||||||
|
|
||||||
|
if source == "igdb" and game.igdb_id:
|
||||||
|
data = lookup_game_from_igdb(str(game.igdb_id))
|
||||||
|
time.sleep(sleep_secs)
|
||||||
|
if not data:
|
||||||
|
continue
|
||||||
|
url = data.get("cover_url" if field_name == "cover" else "screenshot_url")
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
r = requests.get(url)
|
||||||
|
if r.status_code != 200:
|
||||||
|
continue
|
||||||
|
fname = f"{game.title}_{game.uuid}.jpg"
|
||||||
|
getattr(game, field_name).save(fname, ContentFile(r.content), save=True)
|
||||||
|
stats["images_fixed"] += 1
|
||||||
|
self.stdout.write(f" [IMAGE] {game} — {field_name} refetched from IGDB")
|
||||||
|
|
||||||
|
elif source == "hltb" and game.hltb_id:
|
||||||
|
data = lookup_game_from_hltb(str(game.hltb_id))
|
||||||
|
time.sleep(sleep_secs)
|
||||||
|
if not data:
|
||||||
|
continue
|
||||||
|
url = data.get("cover_url")
|
||||||
|
if not url:
|
||||||
|
continue
|
||||||
|
headers = {"User-Agent": "Vrobbler 0.11.12"}
|
||||||
|
r = requests.get(url, headers=headers)
|
||||||
|
if r.status_code != 200:
|
||||||
|
continue
|
||||||
|
fname = f"{game.title}_cover_{game.uuid}.jpg"
|
||||||
|
game.hltb_cover.save(fname, ContentFile(r.content), save=True)
|
||||||
|
stats["images_fixed"] += 1
|
||||||
|
self.stdout.write(f" [IMAGE] {game} — hltb_cover refetched from HLtB")
|
||||||
|
|
||||||
|
return stats
|
||||||
|
|||||||
Reference in New Issue
Block a user