Files
vrobbler/vrobbler/apps/videos/models.py
Colin Powell b4180afbed
Some checks failed
build / test (push) Has been cancelled
[videos] Fix metadata for series script
2026-06-17 12:05:22 -04:00

711 lines
24 KiB
Python

import json
import logging
import re
from dataclasses import dataclass
from typing import Optional
from uuid import uuid4
import requests
from django.conf import settings
from django.core.files.base import ContentFile
from django.db import models
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from django_extensions.db.models import TimeStampedModel
from imagekit.models import ImageSpecField
from imagekit.processors import ResizeToFit
from scrobbles.mixins import (
ObjectWithGenres,
ScrobblableConstants,
ScrobblableMixin,
)
from taggit.managers import TaggableManager
from videos.metadata import VideoMetadata
from videos.sources.omdb import lookup_video_from_omdb
from videos.sources.tmdb import lookup_video_from_tmdb
from videos.sources.youtube import lookup_video_from_youtube
from vrobbler.apps.scrobbles.dataclasses import BaseLogData, WithPeopleLogData
YOUTUBE_VIDEO_URL = "https://www.youtube.com/watch?v="
YOUTUBE_CHANNEL_URL = "https://www.youtube.com/channel/"
YOUTUBE_ID_PATTERN = re.compile(r"^[A-Za-z0-9_-]{11}$")
IMDB_VIDEO_URL = "https://www.imdb.com/title/"
logger = logging.getLogger(__name__)
BNULL = {"blank": True, "null": True}
@dataclass
class VideoLogData(BaseLogData, WithPeopleLogData):
rating: Optional[int] = None
class Channel(ScrobblableMixin):
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
name = models.CharField(max_length=255)
cover_image = models.ImageField(upload_to="videos/channels/", **BNULL)
cover_small = ImageSpecField(
source="cover_image",
processors=[ResizeToFit(100, 100)],
format="JPEG",
options={"quality": 60},
)
cover_medium = ImageSpecField(
source="cover_image",
processors=[ResizeToFit(300, 300)],
format="JPEG",
options={"quality": 75},
)
youtube_id = models.CharField(max_length=255, **BNULL)
twitch_id = models.CharField(max_length=255, **BNULL)
description = models.TextField(**BNULL)
custom_url = models.CharField(max_length=255, **BNULL)
genre = TaggableManager(through=ObjectWithGenres, blank=True, verbose_name="Genre")
class Meta:
verbose_name_plural = "channels"
SECONDS_TO_STALE = 7200 # 2 hours
COMPLETION_PERCENT = 100
def __str__(self) -> str:
return self.name
@property
def run_time_seconds(self) -> int:
return self.base_run_time_seconds or 7200
@property
def title(self):
return self.name
@property
def safe_cover_image_url(self) -> str:
if self.cover_image:
try:
if self.cover_image.storage.exists(self.cover_image.name):
return self.cover_medium.url
except Exception:
pass
return "/static/images/not-found.jpg"
@property
def youtube_url(self) -> str:
if self.youtube_id:
return YOUTUBE_CHANNEL_URL + self.youtube_id
return ""
@property
def twitch_url(self) -> str:
if self.twitch_id:
return f"https://www.twitch.tv/{self.twitch_id}"
return ""
def save_image_from_url(self, url: str, force_update: bool = False):
if not self.cover_image or (force_update and url):
r = requests.get(url)
if r.status_code == 200:
fname = f"{self.name}_{self.uuid}.jpg"
self.cover_image.save(fname, ContentFile(r.content), save=True)
def scrobbles_for_user(self, user_id: int, include_playing=False):
from scrobbles.models import Scrobble
played_query = models.Q(played_to_completion=True)
if include_playing:
played_query = models.Q()
return Scrobble.objects.filter(
played_query,
models.Q(channel=self) | models.Q(video__channel=self),
user=user_id,
).order_by("-timestamp")
def get_absolute_url(self):
return reverse("videos:channel_detail", kwargs={"slug": self.uuid})
@classmethod
def find_or_create(cls, twitch_handle: str, overwrite: bool = False):
channel, created = cls.objects.get_or_create(twitch_id=twitch_handle)
if not created and not overwrite:
return channel
try:
from videos.sources.twitch import lookup_channel_from_twitch
metadata = lookup_channel_from_twitch(twitch_handle)
if metadata.name:
channel.name = metadata.name
if metadata.profile_image_url:
channel.save_image_from_url(metadata.profile_image_url)
channel.save()
except Exception as e:
logger.warning(f"Failed to get channel metadata: {e}")
return channel
def scrobble_for_user(
self,
user_id: int,
status: str = "started",
source: str = "Twitch",
**kwargs,
):
from scrobbles.models import Scrobble
return Scrobble.create_or_update(
media_object=self,
user_id=user_id,
status=status,
source=source,
**kwargs,
)
def fix_metadata(self, force: bool = False):
if self.youtube_id:
GOOGLE_CHANNELS_URL = "https://www.googleapis.com/youtube/v3/channels?part=snippet,topicDetails&id={channel_id}&key={key}"
url = GOOGLE_CHANNELS_URL.format(
channel_id=self.youtube_id,
key=settings.GOOGLE_API_KEY,
)
headers = {"User-Agent": "Vrobbler 0.11.12"}
response = requests.get(url, headers=headers)
if response.status_code != 200:
logger.warning(
"Bad response from Google for channel",
extra={"response": response},
)
return
items = json.loads(response.content).get("items", [])
if not items:
logger.warning(f"No YouTube channel data for {self.youtube_id}")
return
snippet = items[0].get("snippet", {})
channel_name = snippet.get("title", "")
if channel_name and (not self.name or force):
self.name = channel_name
channel_description = snippet.get("description", "")
if channel_description and (not self.description or force):
self.description = channel_description
custom_url = snippet.get("customUrl", "")
if custom_url and (not self.custom_url or force):
self.custom_url = custom_url
thumbnails = snippet.get("thumbnails", {})
cover_url = (
thumbnails.get("high", {}).get("url")
or thumbnails.get("medium", {}).get("url")
or thumbnails.get("default", {}).get("url")
)
if cover_url:
self.save_image_from_url(cover_url, force_update=force)
topic_details = items[0].get("topicDetails", {})
topic_categories = topic_details.get("topicCategories", [])
if topic_categories:
if force:
self.genre.clear()
for category_url in topic_categories:
topic_name = category_url.rstrip("/").split("/")[-1]
topic_name = topic_name.replace("_", " ").replace("-", " ")
self.genre.add(topic_name)
self.save()
return
if self.twitch_id:
from videos.sources.twitch import lookup_channel_from_twitch
metadata = lookup_channel_from_twitch(self.twitch_id)
if metadata.name and (not self.name or force):
self.name = metadata.name
if metadata.profile_image_url:
self.save_image_from_url(metadata.profile_image_url, force_update=force)
self.save()
return
logger.warning(f"No youtube_id or twitch_id set for channel {self}")
return
class Series(TimeStampedModel):
uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
name = models.CharField(max_length=255)
plot = models.TextField(**BNULL)
imdb_id = models.CharField(max_length=20, **BNULL)
imdb_rating = models.FloatField(**BNULL)
cover_image = models.ImageField(upload_to="videos/series/", **BNULL)
cover_small = ImageSpecField(
source="cover_image",
processors=[ResizeToFit(100, 100)],
format="JPEG",
options={"quality": 60},
)
cover_medium = ImageSpecField(
source="cover_image",
processors=[ResizeToFit(300, 300)],
format="JPEG",
options={"quality": 75},
)
preferred_source = models.CharField(max_length=100, **BNULL)
genre = TaggableManager(through=ObjectWithGenres, blank=True, verbose_name="Genre")
class Meta:
verbose_name_plural = "series"
def __str__(self):
return self.name
def get_absolute_url(self):
return reverse("videos:series_detail", kwargs={"slug": self.uuid})
def imdb_link(self) -> str:
if self.imdb_id:
return IMDB_VIDEO_URL + self.imdb_id
return ""
@property
def primary_image_url(self) -> str:
url = ""
if self.cover_image:
url = self.cover_image_medium.url
return url
@property
def safe_cover_image_url(self) -> str:
if self.cover_image:
try:
if self.cover_image.storage.exists(self.cover_image.name):
return self.cover_medium.url
except Exception:
pass
return "/static/images/not-found.jpg"
def save_image_from_url(self, url: str, force_update: bool = False):
if not self.cover_image or (force_update and url):
r = requests.get(url)
if r.status_code == 200:
fname = f"{self.name}_{self.uuid}.jpg"
self.cover_image.save(fname, ContentFile(r.content), save=True)
def scrobbles_for_user(self, user_id: int, include_playing=False):
from scrobbles.models import Scrobble
played_query = models.Q(played_to_completion=True)
if include_playing:
played_query = models.Q()
return Scrobble.objects.filter(
played_query,
video__tv_series=self,
user=user_id,
).order_by("-timestamp")
def last_scrobbled_episode(self, user_id: int) -> Optional["Video"]:
episode = None
last_scrobble = self.scrobbles_for_user(user_id, include_playing=True).first()
if last_scrobble:
episode = last_scrobble.media_obj
return episode
def is_episode_playing(self, user_id: int) -> bool:
last_scrobble = self.scrobbles_for_user(user_id, include_playing=True).first()
if not last_scrobble:
return False
return not last_scrobble.played_to_completion
def fix_metadata(self, force_update=False):
from tmdbv3api import TV
if not self.imdb_id:
tv = TV()
results = tv.search(self.name)
if results:
show_id = results[0].id
external_ids = tv.external_ids(show_id)
if external_ids and external_ids.imdb_id:
self.imdb_id = external_ids.imdb_id
self.save(update_fields=["imdb_id"])
else:
logger.warning(f"No IMDB ID found on TMDB for {self}")
return
else:
logger.warning(f"No results on TMDB for {self.name}")
return
video_metadata = lookup_video_from_tmdb(self.imdb_id)
if not video_metadata or not video_metadata.title:
logger.warning(f"No metadata for {self}")
return
if video_metadata.cover_url and (not self.cover_image or force_update):
r = requests.get(video_metadata.cover_url)
if r.status_code == 200:
fname = f"{self.name}_{self.uuid}.jpg"
self.cover_image.save(fname, ContentFile(r.content), save=True)
self.plot = video_metadata.plot or ""
self.imdb_rating = getattr(video_metadata, "imdb_rating", None)
self.save()
if video_metadata.genres:
self.genre.add(*video_metadata.genres)
@classmethod
def find_or_create(cls, imdb_id: str, overwrite: bool = True):
series, created = cls.objects.get_or_create(imdb_id=imdb_id)
if not created and not overwrite:
logger.info("Series not created and overwrite=False, returning")
return series
try:
metadata = lookup_video_from_tmdb(imdb_id)
except Exception as e:
logger.warning(f"TMDB lookup failed for series {imdb_id}: {e}")
metadata = None
if not metadata or not metadata.title:
metadata = lookup_video_from_omdb(imdb_id)
if not metadata or not metadata.title:
logger.warning(f"No metadata found for series {imdb_id} from TMDB or OMDB")
return series
vdict, _, cover, genres = metadata.as_dict_with_cover_and_genres()
vdict.pop("video_type")
vdict["name"] = vdict.pop("title")
for k, v in vdict.items():
setattr(series, k, v)
series.save()
if cover:
series.save_image_from_url(cover)
if genres:
series.genre.add(*genres)
return series
class Video(ScrobblableMixin):
COMPLETION_PERCENT = getattr(settings, "VIDEO_COMPLETION_PERCENT", 90)
SECONDS_TO_STALE = getattr(settings, "VIDEO_SECONDS_TO_STALE", 14400)
METADATA_CLASS = VideoMetadata
class VideoType(models.TextChoices):
UNKNOWN = "U", _("Unknown")
TV_EPISODE = "E", _("TV Episode")
MOVIE = "M", _("Movie")
SKATE_VIDEO = "S", _("Skate Video")
YOUTUBE = "Y", _("YouTube Video")
TWITCH = "T", _("Twitch Video")
video_type = models.CharField(
max_length=1,
choices=VideoType.choices,
default=VideoType.UNKNOWN,
)
overview = models.TextField(**BNULL)
tagline = models.TextField(**BNULL)
year = models.IntegerField(**BNULL)
# TV show specific fields
tv_series = models.ForeignKey(Series, on_delete=models.DO_NOTHING, **BNULL)
channel = models.ForeignKey(Channel, on_delete=models.DO_NOTHING, **BNULL)
season_number = models.IntegerField(**BNULL)
episode_number = models.IntegerField(**BNULL)
next_imdb_id = models.CharField(max_length=20, **BNULL)
imdb_id = models.CharField(max_length=20, **BNULL)
imdb_rating = models.FloatField(**BNULL)
tmdb_rating = models.FloatField(**BNULL)
cover_image = models.ImageField(upload_to="videos/video/", **BNULL)
cover_image_small = ImageSpecField(
source="cover_image",
processors=[ResizeToFit(100, 100)],
format="JPEG",
options={"quality": 60},
)
cover_image_medium = ImageSpecField(
source="cover_image",
processors=[ResizeToFit(300, 300)],
format="JPEG",
options={"quality": 75},
)
tvrage_id = models.CharField(max_length=20, **BNULL)
tvdb_id = models.CharField(max_length=20, **BNULL)
tmdb_id = models.CharField(max_length=20, **BNULL)
youtube_id = models.CharField(max_length=255, **BNULL)
twitch_id = models.CharField(max_length=255, **BNULL)
plot = models.TextField(**BNULL)
upload_date = models.DateField(**BNULL)
def __str__(self):
if not self.title:
return self.youtube_id or self.imdb_id
if self.video_type == self.VideoType.TV_EPISODE:
return f"{self.title} / [S{self.season_number}E{self.episode_number}] {self.tv_series}"
if self.video_type == self.VideoType.YOUTUBE:
return f"{self.title} / {self.channel}"
return self.title
class Meta:
constraints = [
models.UniqueConstraint(
fields=["imdb_id", "youtube_id"],
name="unique_imdb_or_youtube_id",
condition=~models.Q(imdb_id__isnull=True)
| ~models.Q(youtube_id__isnull=True),
),
]
def get_absolute_url(self):
return reverse("videos:video_detail", kwargs={"slug": self.uuid})
@property
def logdata_cls(self):
return VideoLogData
@property
def subtitle(self):
if self.tv_series:
return self.tv_series
return ""
@property
def imdb_link(self):
prefix = ""
if "tt" not in self.imdb_id:
prefix = "tt"
return f"https://www.imdb.com/title/{prefix}{self.imdb_id}"
@property
def info_link(self):
return self.imdb_link
@property
def link(self) -> str:
return self.imdb_link
@property
def youtube_link(self) -> str:
if self.youtube_id:
return f"https://www.youtube.com/watch?v={self.youtube_id}"
return ""
@property
def primary_image_url(self) -> str:
url = ""
if self.cover_image:
url = self.cover_image_medium.url
return url
@property
def safe_cover_image_url(self) -> str:
if self.cover_image:
try:
if self.cover_image.storage.exists(self.cover_image.name):
return self.cover_image_medium.url
except Exception:
pass
return "/static/images/not-found.jpg"
@property
def strings(self) -> ScrobblableConstants:
return ScrobblableConstants(verb="Watching", tags="movie_camera")
def save_image_from_url(self, url: str, force_update: bool = False):
if not self.cover_image or (force_update and url):
r = requests.get(url)
if r.status_code == 200:
fname = f"{self.title}_{self.uuid}.jpg"
self.cover_image.save(fname, ContentFile(r.content), save=True)
def fix_metadata(self, force_update: bool = False) -> None:
if self.video_type == self.VideoType.YOUTUBE and self.youtube_id:
vdict, _, cover, genres = lookup_video_from_youtube(
self.youtube_id
).as_dict_with_cover_and_genres()
for k, v in vdict.items():
setattr(self, k, v)
self.save()
if cover:
self.save_image_from_url(cover)
if genres:
self.genre.add(*genres)
return
if self.video_type == self.VideoType.TWITCH and self.twitch_id:
from videos.sources.twitch import lookup_video_from_twitch
metadata = lookup_video_from_twitch(self.twitch_id)
self.title = metadata.title or f"Twitch VOD {self.twitch_id}"
self.overview = metadata.overview
self.base_run_time_seconds = metadata.base_run_time_seconds
if metadata.upload_date:
self.upload_date = metadata.upload_date
if metadata.year:
self.year = metadata.year
self.video_type = Video.VideoType.TWITCH
if metadata.channel_id:
from videos.models import Channel
self.channel = Channel.objects.filter(id=metadata.channel_id).first()
self.save()
if metadata.cover_url:
self.save_image_from_url(metadata.cover_url)
return
if self.imdb_id:
try:
metadata = lookup_video_from_tmdb(self.imdb_id)
except Exception as e:
logger.warning(f"TMDB lookup failed for {self.imdb_id}: {e}")
metadata = None
if not metadata or not metadata.title:
metadata = lookup_video_from_omdb(self.imdb_id)
if not metadata or not metadata.title:
logger.warning(f"No metadata found for {self} from TMDB or OMDB")
return
vdict, series_id, cover, genres = metadata.as_dict_with_cover_and_genres()
for k, v in vdict.items():
setattr(self, k, v)
if series_id:
self.tv_series = Series.find_or_create(imdb_id=series_id)
self.save()
if cover:
self.save_image_from_url(cover)
if genres:
self.genre.add(*genres)
return
logger.warning(
f"No metadata source available for {self} (type={self.video_type})"
)
@classmethod
def get_from_youtube_id(cls, youtube_id: str, overwrite: bool = False) -> "Video":
video, created = cls.objects.get_or_create(youtube_id=youtube_id)
if not created and not overwrite:
return video
vdict, _, cover, genres = lookup_video_from_youtube(
youtube_id
).as_dict_with_cover_and_genres()
if created or overwrite:
for k, v in vdict.items():
setattr(video, k, v)
video.save()
if cover:
video.save_image_from_url(cover)
if genres:
video.genre.add(*genres)
return video
@classmethod
def get_from_imdb_id(cls, imdb_id: str, overwrite: bool = False) -> "Video":
video, created = cls.objects.get_or_create(imdb_id=imdb_id)
if not created and not overwrite:
return video
try:
metadata = lookup_video_from_tmdb(imdb_id)
except Exception as e:
logger.warning(f"TMDB lookup failed for {imdb_id}: {e}")
metadata = None
if not metadata or not metadata.title:
metadata = lookup_video_from_omdb(imdb_id)
if not metadata or not metadata.title:
logger.warning(f"No metadata found for {imdb_id} from TMDB or OMDB")
return video
vdict, series_id, cover, genres = metadata.as_dict_with_cover_and_genres()
if created or overwrite:
for k, v in vdict.items():
setattr(video, k, v)
if series_id:
video.tv_series = Series.find_or_create(imdb_id=series_id)
video.save()
if cover:
video.save_image_from_url(cover)
if genres:
video.genre.add(*genres)
return video
@classmethod
def get_from_twitch_id(cls, twitch_id: str, overwrite: bool = False) -> "Video":
video, created = cls.objects.get_or_create(twitch_id=twitch_id)
if not created and not overwrite:
return video
from videos.sources.twitch import lookup_video_from_twitch
metadata = lookup_video_from_twitch(twitch_id)
if created or overwrite:
video.title = metadata.title or f"Twitch VOD {twitch_id}"
video.overview = metadata.overview
video.cover_url = metadata.cover_url
video.base_run_time_seconds = metadata.base_run_time_seconds
if metadata.upload_date:
video.upload_date = metadata.upload_date
if metadata.year:
video.year = metadata.year
video.video_type = Video.VideoType.TWITCH
if metadata.channel_id:
from videos.models import Channel
video.channel = Channel.objects.filter(id=metadata.channel_id).first()
video.save()
if metadata.cover_url:
video.save_image_from_url(metadata.cover_url)
return video
@classmethod
def find_or_create(
cls, source_id: Optional[str], overwrite: bool = False
) -> Optional["Video"]:
if not source_id:
logger.warning("No source_id provided for Video.find_or_create")
return None
if "tt" in source_id:
return cls.get_from_imdb_id(source_id, overwrite)
if bool(YOUTUBE_ID_PATTERN.match(source_id)):
return cls.get_from_youtube_id(source_id, overwrite)
if source_id.isdigit():
return cls.get_from_twitch_id(source_id, overwrite)
logger.warning("Video ID not recognized, not scrobbling")
return None