From 9140e0cb95e75514828307be11031c004830c9ef Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Tue, 25 Aug 2026 19:31:27 -0400 Subject: [PATCH] [videos] Merge TMDB + OMDB metadata for reliable run time and ratings --- PROJECT.org | 6 +- .../commands/cleanup_video_metadata.py | 91 +++++++++++++++++++ vrobbler/apps/videos/metadata.py | 60 +++++++++++- vrobbler/apps/videos/models.py | 60 ++++++------ vrobbler/apps/videos/sources/tmdb.py | 14 ++- 5 files changed, 192 insertions(+), 39 deletions(-) create mode 100644 vrobbler/apps/videos/management/commands/cleanup_video_metadata.py diff --git a/PROJECT.org b/PROJECT.org index 66a2c9e..c7379f8 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -18,7 +18,7 @@ tasks, Todoist tasks, web pages I've read and trails I've hiked has turned out to be sometimes cathartic and sometimes functional as I try to remember when I did a thing. -* Backlog [0/30] :vrobbler:project:personal: +* Backlog [0/31] :vrobbler:project:personal: ** TODO [#C] Configure IMAP folder/start in user profile :imap:settings: *** Description @@ -590,4 +590,8 @@ The Edit log form should have from top to bottom: - People (which should be similar to the Bird widget on BirdLocation and allow setting per user score, win true/false, rank, new true/false, seat_ordrer) - Expansion ids (which should a multi-select widget of expansions for this game) - Location (which should be a drop down of BoardGameLocations for this user) +** STRT [#B] Video metadata is not getting base run time seconds reliably :videos:metadata: +:PROPERTIES: +:ID: cdb7cf49-a9df-3368-7cf8-12ee0775a3ce +:END: diff --git a/vrobbler/apps/videos/management/commands/cleanup_video_metadata.py b/vrobbler/apps/videos/management/commands/cleanup_video_metadata.py new file mode 100644 index 0000000..147b188 --- /dev/null +++ b/vrobbler/apps/videos/management/commands/cleanup_video_metadata.py @@ -0,0 +1,91 @@ +import logging + +from django.core.management.base import BaseCommand +from django.db import models, transaction + +logger = logging.getLogger(__name__) + +BOGUS_RUN_TIME_SECONDS = 900 + + +class Command(BaseCommand): + help = "Enrich video metadata (run time, ratings) from TMDB/OMDB APIs" + + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Overwrite existing cover image", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without making changes", + ) + parser.add_argument( + "--imdb-id", + type=str, + help="Only process videos with this imdb_id", + ) + parser.add_argument( + "--all", + action="store_true", + help="Process every video with an imdb_id, not just ones missing metadata", + ) + + def handle(self, *args, **options): + from videos.models import Video + + force = options["force"] + dry_run = options["dry_run"] + imdb_id = options["imdb_id"] + process_all = options["all"] + + qs = Video.objects.filter(imdb_id__isnull=False).exclude(imdb_id="") + if imdb_id: + qs = qs.filter(imdb_id=imdb_id) + if not process_all: + qs = qs.filter( + models.Q(base_run_time_seconds__isnull=True) + | models.Q(base_run_time_seconds=BOGUS_RUN_TIME_SECONDS) + | models.Q(imdb_rating__isnull=True) + | models.Q(tmdb_rating__isnull=True) + ) + + videos = list(qs) + total = len(videos) + self.stdout.write(f"Total videos to process: {total}") + + if dry_run: + for video in videos: + label = video.title or str(video.uuid) + runtime = video.base_run_time_seconds + if runtime == BOGUS_RUN_TIME_SECONDS: + runtime = "900 (bogus)" + status = ( + f"runtime={runtime}, imdb_rating={video.imdb_rating}, " + f"tmdb_rating={video.tmdb_rating}" + ) + self.stdout.write(f" [DRY RUN] Would fix {label} ({status})") + return + + updated = 0 + errors = 0 + for video in videos: + label = video.title or str(video.uuid) + try: + with transaction.atomic(): + video.fix_metadata(force_update=force) + updated += 1 + self.stdout.write(f" [{updated}/{total}] {label}") + except Exception as e: + errors += 1 + self.stdout.write( + self.style.ERROR( + f" Error updating {label} (imdb_id={video.imdb_id}): {e}" + ) + ) + + self.stdout.write( + self.style.SUCCESS(f"\nDone! {updated} videos updated, {errors} errors") + ) diff --git a/vrobbler/apps/videos/metadata.py b/vrobbler/apps/videos/metadata.py index 487ffbc..9eb0340 100644 --- a/vrobbler/apps/videos/metadata.py +++ b/vrobbler/apps/videos/metadata.py @@ -34,9 +34,7 @@ class VideoMetadata: title: str video_type: VideoType = VideoType.UNKNOWN - base_run_time_seconds: int = ( - 60 # Silly default, but things break if this is 0 or null - ) + base_run_time_seconds: Optional[int] = None imdb_id: Optional[str] tmdb_id: Optional[str] youtube_id: Optional[str] @@ -69,7 +67,7 @@ class VideoMetadata: imdb_id: Optional[str] = "", youtube_id: Optional[str] = "", twitch_id: Optional[str] = "", - base_run_time_seconds: int = 900, + base_run_time_seconds: Optional[int] = None, ): self.title = "" self.imdb_id = imdb_id @@ -87,3 +85,57 @@ class VideoMetadata: if "tv_series_imdb_id" in video_dict.keys(): series_id = video_dict.pop("tv_series_imdb_id") return video_dict, series_id, cover, genres + + +MERGE_FIELDS = ( + "title", + "video_type", + "base_run_time_seconds", + "imdb_id", + "tmdb_id", + "youtube_id", + "twitch_id", + "episode_number", + "season_number", + "next_imdb_id", + "year", + "plot", + "overview", + "imdb_rating", + "tmdb_rating", + "cover_url", + "tv_series_imdb_id", +) + + +def merge_metadata( + primary: Optional[VideoMetadata], + secondary: Optional[VideoMetadata], +) -> Optional[VideoMetadata]: + """Merge two metadata lookups, filling gaps in ``primary`` from ``secondary``. + + ``primary`` (typically the TMDB result) wins; ``secondary`` (typically the + OMDB result) only fills fields the primary source did not set. + """ + if not primary: + return secondary + if not secondary: + return primary + + for field in MERGE_FIELDS: + current = getattr(primary, field, None) + if field == "video_type": + if current != VideoType.UNKNOWN: + continue + elif current not in (None, ""): + continue + value = getattr(secondary, field, None) + if value in (None, ""): + continue + setattr(primary, field, value) + + secondary_genres = getattr(secondary, "genres", None) + if not getattr(primary, "genres", None) and secondary_genres: + primary.genres = secondary_genres + + return primary diff --git a/vrobbler/apps/videos/models.py b/vrobbler/apps/videos/models.py index b4905da..92bc6b7 100644 --- a/vrobbler/apps/videos/models.py +++ b/vrobbler/apps/videos/models.py @@ -21,7 +21,7 @@ from scrobbles.mixins import ( ScrobblableMixin, ) from taggit.managers import TaggableManager -from videos.metadata import VideoMetadata +from videos.metadata import VideoMetadata, merge_metadata from videos.sources.omdb import lookup_video_from_omdb from videos.sources.tmdb import lookup_video_from_tmdb from videos.sources.youtube import lookup_video_from_youtube @@ -37,6 +37,17 @@ logger = logging.getLogger(__name__) BNULL = {"blank": True, "null": True} +def lookup_merged_video_metadata(imdb_id: str) -> Optional[VideoMetadata]: + """Look a title up on both TMDB and OMDB, merging the results.""" + try: + tmdb_metadata = lookup_video_from_tmdb(imdb_id) + except Exception as e: + logger.warning(f"TMDB lookup failed for {imdb_id}: {e}") + tmdb_metadata = None + omdb_metadata = lookup_video_from_omdb(imdb_id) + return merge_metadata(tmdb_metadata, omdb_metadata) + + @dataclass class VideoLogData(BaseLogData, WithPeopleLogData): rating: Optional[int] = None @@ -341,22 +352,26 @@ class Series(TimeStampedModel): logger.warning(f"No results on TMDB for {self.name}") return - video_metadata = lookup_video_from_tmdb(self.imdb_id) + video_metadata = lookup_merged_video_metadata(self.imdb_id) if not video_metadata or not video_metadata.title: logger.warning(f"No metadata for {self}") return - if video_metadata.cover_url and (not self.cover_image or force_update): + if getattr(video_metadata, "cover_url", None) and ( + not self.cover_image or force_update + ): r = requests.get(video_metadata.cover_url) if r.status_code == 200: fname = f"{self.name}_{self.uuid}.jpg" self.cover_image.save(fname, ContentFile(r.content), save=True) - self.plot = video_metadata.plot or "" - self.imdb_rating = getattr(video_metadata, "imdb_rating", None) + self.plot = getattr(video_metadata, "plot", None) or "" + self.imdb_rating = ( + getattr(video_metadata, "imdb_rating", None) or self.imdb_rating + ) self.save() - if video_metadata.genres: + if getattr(video_metadata, "genres", None): self.genre.add(*video_metadata.genres) @classmethod @@ -367,14 +382,7 @@ class Series(TimeStampedModel): logger.info("Series not created and overwrite=False, returning") return series - try: - metadata = lookup_video_from_tmdb(imdb_id) - except Exception as e: - logger.warning(f"TMDB lookup failed for series {imdb_id}: {e}") - metadata = None - - if not metadata or not metadata.title: - metadata = lookup_video_from_omdb(imdb_id) + metadata = lookup_merged_video_metadata(imdb_id) if not metadata or not metadata.title: logger.warning(f"No metadata found for series {imdb_id} from TMDB or OMDB") @@ -385,6 +393,8 @@ class Series(TimeStampedModel): vdict["name"] = vdict.pop("title") for k, v in vdict.items(): + if v is None: + continue setattr(series, k, v) series.save() @@ -570,14 +580,7 @@ class Video(ScrobblableMixin): return if self.imdb_id: - try: - metadata = lookup_video_from_tmdb(self.imdb_id) - except Exception as e: - logger.warning(f"TMDB lookup failed for {self.imdb_id}: {e}") - metadata = None - - if not metadata or not metadata.title: - metadata = lookup_video_from_omdb(self.imdb_id) + metadata = lookup_merged_video_metadata(self.imdb_id) if not metadata or not metadata.title: logger.warning(f"No metadata found for {self} from TMDB or OMDB") @@ -586,6 +589,8 @@ class Video(ScrobblableMixin): vdict, series_id, cover, genres = metadata.as_dict_with_cover_and_genres() for k, v in vdict.items(): + if v is None: + continue setattr(self, k, v) if series_id: @@ -629,14 +634,7 @@ class Video(ScrobblableMixin): if not created and not overwrite: return video - try: - metadata = lookup_video_from_tmdb(imdb_id) - except Exception as e: - logger.warning(f"TMDB lookup failed for {imdb_id}: {e}") - metadata = None - - if not metadata or not metadata.title: - metadata = lookup_video_from_omdb(imdb_id) + metadata = lookup_merged_video_metadata(imdb_id) if not metadata or not metadata.title: logger.warning(f"No metadata found for {imdb_id} from TMDB or OMDB") @@ -646,6 +644,8 @@ class Video(ScrobblableMixin): if created or overwrite: for k, v in vdict.items(): + if v is None: + continue setattr(video, k, v) if series_id: diff --git a/vrobbler/apps/videos/sources/tmdb.py b/vrobbler/apps/videos/sources/tmdb.py index e34c4fb..1e8a525 100644 --- a/vrobbler/apps/videos/sources/tmdb.py +++ b/vrobbler/apps/videos/sources/tmdb.py @@ -47,7 +47,9 @@ def lookup_video_from_tmdb(name_or_id: str, kind: str = "movie") -> VideoMetadat video_metadata.year = pendulum.parse(media.release_date).year video_metadata.genres = [g.get("name", "") for g in media.genres] video_metadata.tmdb_id = media.id - video_metadata.base_run_time_seconds = media.runtime * 60 + video_metadata.base_run_time_seconds = ( + media.runtime * 60 if media.runtime else None + ) video_metadata.plot = media.overview video_metadata.overview = media.overview video_metadata.tmdb_rating = media.vote_average @@ -56,10 +58,10 @@ def lookup_video_from_tmdb(name_or_id: str, kind: str = "movie") -> VideoMetadat media = TV().details(tmdb_result.tv_results[0].id) video_metadata.video_type = VideoType.TV_EPISODE.value video_metadata.title = media.name - video_metadata.cover_url = ( - TMDB_IMAGE_URL + media.poster_path + video_metadata.cover_url = TMDB_IMAGE_URL + media.poster_path + video_metadata.year = ( + pendulum.parse(media.first_air_date).year if media.first_air_date else None ) - video_metadata.year = pendulum.parse(media.first_air_date).year if media.first_air_date else None video_metadata.genres = [g.get("name", "") for g in media.genres] video_metadata.tmdb_id = media.id video_metadata.base_run_time_seconds = ( @@ -91,6 +93,10 @@ def lookup_video_from_tmdb(name_or_id: str, kind: str = "movie") -> VideoMetadat video_metadata.tmdb_id = media.id video_metadata.plot = media.overview video_metadata.overview = media.overview + video_metadata.tmdb_rating = getattr(media, "vote_average", None) + runtime = getattr(media, "runtime", None) + if runtime: + video_metadata.base_run_time_seconds = runtime * 60 if not media: logger.warning("Video not found on TMDB", extra={"imdb_id": imdb_id})