From 3a50a8b015d8f79233b4dc2a550ca65375c1751d Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Fri, 6 Sep 2024 01:22:54 -0400 Subject: [PATCH] [videos] Fixing imdb lookups and making it more modular --- vrobbler/apps/music/constants.py | 2 + vrobbler/apps/scrobbles/models.py | 24 ++++++++ vrobbler/apps/scrobbles/scrobblers.py | 84 ++++++++++++++++++++++++--- vrobbler/apps/videos/imdb.py | 65 ++++++++++++++++----- vrobbler/apps/videos/models.py | 23 ++++---- vrobbler/apps/videos/utils.py | 56 ++++++++++-------- 6 files changed, 199 insertions(+), 55 deletions(-) diff --git a/vrobbler/apps/music/constants.py b/vrobbler/apps/music/constants.py index 6d70d02..35040be 100644 --- a/vrobbler/apps/music/constants.py +++ b/vrobbler/apps/music/constants.py @@ -19,6 +19,8 @@ JELLYFIN_POST_KEYS = { "ALBUM_NAME": "Album", "ARTIST_NAME": "Artist", "STATUS": "Status", + "VIDEO_TITLE": "Name", + "IMDB_ID": "Provider_imdb", } MOPIDY_POST_KEYS = { diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index c190da1..01e034a 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -769,6 +769,30 @@ class Scrobble(TimeStampedModel): if not padding_seconds: return is_in_progress + if not self.media_obj: + logger.info( + "[scrobbling] scrobble has no media obj", + extra={ + "media_id": self.media_obj, + "scrobble_id": self.id, + "media_type": self.media_type, + "probably_still_in_progress": is_in_progress, + }, + ) + return is_in_progress + + if not self.media_obj.run_time_seconds: + logger.info( + "[scrobbling] media has no run time seconds, cannot calculate end", + extra={ + "media_id": self.media_obj.id, + "scrobble_id": self.id, + "media_type": self.media_type, + "probably_still_in_progress": is_in_progress, + }, + ) + return is_in_progress + expected_end = self.timestamp + datetime.timedelta( seconds=self.media_obj.run_time_seconds ) diff --git a/vrobbler/apps/scrobbles/scrobblers.py b/vrobbler/apps/scrobbles/scrobblers.py index a444b52..d0ecc42 100644 --- a/vrobbler/apps/scrobbles/scrobblers.py +++ b/vrobbler/apps/scrobbles/scrobblers.py @@ -65,7 +65,7 @@ def jellyfin_scrobble_media( media_type = Scrobble.MediaType.TRACK logger.info( - "[jellyfin_scrobble_track] called", + "[jellyfin_scrobble_media] called", extra={ "user_id": user_id, "post_data": post_data, @@ -81,32 +81,45 @@ def jellyfin_scrobble_media( # Jellyfin has some race conditions with it's webhooks, these hacks fix some of them if null_position_on_progress: logger.info( - "[jellyfin_scrobble_track] no playback position tick, aborting", + "[jellyfin_scrobble_media] no playback position tick, aborting", extra={"post_data": post_data}, ) return + timestamp = parse( + post_data.get(JELLYFIN_POST_KEYS.get("TIMESTAMP")) + ).replace(tzinfo=pytz.utc) + if media_type == Scrobble.MediaType.VIDEO: media_obj = Video.find_or_create(post_data) + playback_position_seconds = (timezone.now() - timestamp).seconds else: media_obj = get_or_create_track( post_data, post_keys=JELLYFIN_POST_KEYS ) + playback_position_seconds = 0 + + if not media_obj: + logger.info( + "[jellyfin_scrobble_media] no video found from POST data", + extra={"post_data": post_data}, + ) + return - timestamp = parse( - post_data.get(JELLYFIN_POST_KEYS.get("TIMESTAMP")) - ).replace(tzinfo=pytz.utc) playback_status = "resumed" if post_data.get("IsPaused"): playback_status = "paused" elif post_data.get("NotificationType") == "PlaybackStop": playback_status = "stopped" - # TODO Add some logging here, maybe? + logger.info( + "[jellyfin_scrobble_media] no playback position tick, aborting", + extra={"post_data": post_data, "playback_status": playback_status}, + ) return media_obj.scrobble_for_user( user_id, - playback_position_seconds=(timezone.now() - timestamp).seconds, + playback_position_seconds=playback_position_seconds, status=playback_status, ) @@ -150,6 +163,17 @@ def manual_scrobble_video_game(hltb_id: str, user_id: int): game = VideoGame.objects.filter(hltb_id=hltb_id).first() if not game: data_dict = lookup_game_from_hltb(hltb_id) + if not data_dict: + logger.info( + "[manual_scrobble_video_game] game not found on hltb", + extra={ + "hltb_id": hltb_id, + "user_id": user_id, + "media_type": Scrobble.MediaType.VIDEO_GAME, + }, + ) + return + game = VideoGame.find_or_create(data_dict) scrobble_dict = { @@ -295,3 +319,49 @@ def gpslogger_scrobble_location(data_dict: dict, user_id: int) -> Scrobble: ) return scrobble + + +def web_scrobbler_scrobble_video_or_song( + data_dict: dict, user_id: Optional[int] +) -> Scrobble: + # We're not going to create music tracks, because the only time + # we'd hit this is if we're listening to a concert or something. + artist_name = data_dict.get("artist") + track_name = data_dict.get("track") + tracks = Track.objects.filter( + artist__name=data_dict.get("artist"), title=data_dict.get("track") + ) + if tracks.count() > 1: + logger.warning( + "Multiple tracks found for Web Scrobbler", + extra={"artist": artist_name, "track": track_name}, + ) + track = tracks.first() + + # No track found, create a Video + if not track: + Video.find_or_create(data_dict) + + # Now we run off a scrobble + mopidy_data = { + "user_id": user_id, + "timestamp": timezone.now(), + "playback_position_seconds": data_dict.get("playback_time_ticks"), + "source": "Mopidy", + "mopidy_status": data_dict.get("status"), + } + + logger.info( + "[scrobblers] webhook mopidy scrobble request received", + extra={ + "episode_id": episode.id if episode else None, + "user_id": user_id, + "scrobble_dict": mopidy_data, + "media_type": Scrobble.MediaType.PODCAST_EPISODE, + }, + ) + + scrobble = None + if episode: + scrobble = Scrobble.create_or_update(episode, user_id, mopidy_data) + return scrobble diff --git a/vrobbler/apps/videos/imdb.py b/vrobbler/apps/videos/imdb.py index 17227e6..1dc887f 100644 --- a/vrobbler/apps/videos/imdb.py +++ b/vrobbler/apps/videos/imdb.py @@ -3,19 +3,21 @@ from django.utils import timezone from imdb import Cinemagoer, helpers from imdb.Character import IMDbParserError +from scrobbles.dataclasses import VideoLogData imdb_client = Cinemagoer() logger = logging.getLogger(__name__) -def lookup_video_from_imdb(name_or_id: str, kind: str = "movie") -> dict: +def lookup_video_from_imdb( + name_or_id: str, kind: str = "movie" +) -> VideoLogData: # Very few video titles start with tt, but IMDB IDs often come in with it if name_or_id.startswith("tt"): name_or_id = name_or_id[2:] - video_dict = {} imdb_id = None try: @@ -23,27 +25,64 @@ def lookup_video_from_imdb(name_or_id: str, kind: str = "movie") -> dict: except ValueError: pass + video_metadata = None if imdb_id: imdb_result = imdb_client.get_movie(name_or_id) - video_dict = imdb_result + imdb_client.update(imdb_result, info=["plot", "synopsis", "taglines"]) + video_metadata = imdb_result - if not video_dict: + if not video_metadata: imdb_results = imdb_client.search_movie(name_or_id) if len(imdb_results) > 1: for result in imdb_results: if result["kind"] == kind: - video_dict = result + video_metadata = result break if len(imdb_results) == 1: - video_dict = imdb_results[0] + video_metadata = imdb_results[0] + imdb_client.update( + video_metadata, info=["plot", "synopsis", "taglines"] + ) - if not video_dict: - logger.warn(f"No video found for key {name_or_id}") - return video_dict - imdb_client.update(video_dict) + if not video_metadata: + logger.info( + f"[lookup_video_from_imdb] no video found on imdb", + extra={"name_or_id": name_or_id}, + ) + return video_metadata - cover_url = video_dict.get("cover url") + imdb_client.update(video_metadata) + + cover_url = video_metadata.get("cover url") if cover_url: - video_dict["cover url"] = helpers.resizeImage(cover_url, width=800) - return video_dict + cover_url = helpers.resizeImage(cover_url, width=800) + + from videos.models import Video + + video_type = Video.VideoType.MOVIE + series_name = None + if video_metadata.get("kind") == "episode": + series_name = video_metadata.get("episode of", None).data.get( + "title", None + ) + video_type = Video.VideoType.TV_EPISODE + + run_time_seconds = 0 + if video_metadata.get("runtimes"): + run_time_seconds = int(video_metadata.get("runtimes")[0]) * 60 + + return { + "title": video_metadata.get("title"), + "imdb_id": video_metadata.get("imdbID"), + "video_type": video_type, + "run_time_seconds": run_time_seconds, + "episode_number": video_metadata.get("episode", None), + "season_number": video_metadata.get("season", None), + "next_imdb_id": video_metadata.get("next episode", None), + "year": video_metadata.get("year", None), + "series_name": series_name, + "plot": video_metadata.get("plot outline"), + "imdb_rating": video_metadata.get("rating"), + "cover_url": cover_url, + } diff --git a/vrobbler/apps/videos/models.py b/vrobbler/apps/videos/models.py index 53ff5e8..a2bdbd7 100644 --- a/vrobbler/apps/videos/models.py +++ b/vrobbler/apps/videos/models.py @@ -11,6 +11,7 @@ from django.utils.translation import gettext_lazy as _ from django_extensions.db.models import TimeStampedModel from imagekit.models import ImageSpecField from imagekit.processors import ResizeToFit +from music.constants import JELLYFIN_POST_KEYS from scrobbles.mixins import ObjectWithGenres, ScrobblableMixin from taggit.managers import TaggableManager from videos.imdb import lookup_video_from_imdb @@ -218,8 +219,17 @@ class Video(ScrobblableMixin): if genres := imdb_dict.data.get("genres"): self.genre.add(*genres) + def scrape_cover_from_url( + self, cover_url: str, force_update: bool = False + ): + if not self.cover_image or force_update: + r = requests.get(cover_url) + if r.status_code == 200: + fname = f"{self.title}_{self.uuid}.jpg" + self.cover_image.save(fname, ContentFile(r.content), save=True) + @classmethod - def find_or_create(cls, data_dict: Dict) -> "Video": + def find_or_create(cls, data_dict: Dict) -> Optional["Video"]: """Given a data dict from Jellyfin, does the heavy lifting of looking up the video and, if need, TV Series, creating both if they don't yet exist. @@ -230,16 +240,9 @@ class Video(ScrobblableMixin): get_or_create_video_from_jellyfin, ) - if "NotificationType" not in data_dict.keys(): - name_or_id = data_dict.get("imdb_id") or data_dict.get("title") - video = get_or_create_video(name_or_id) - return video + video = get_or_create_video(data_dict, JELLYFIN_POST_KEYS) - if not data_dict.get("Provider_imdb"): - title = data_dict.get("Name", "") - logger.warn( - f"No IMDB ID from Jellyfin, check metadata for {title}" - ) + if not video: return return get_or_create_video_from_jellyfin(data_dict) diff --git a/vrobbler/apps/videos/utils.py b/vrobbler/apps/videos/utils.py index c732755..c8990e9 100644 --- a/vrobbler/apps/videos/utils.py +++ b/vrobbler/apps/videos/utils.py @@ -7,55 +7,61 @@ from scrobbles.utils import convert_to_seconds logger = logging.getLogger(__name__) -def get_or_create_video(name_or_id: str, force_update=False): - imdb_dict = lookup_video_from_imdb(name_or_id) +def get_or_create_video(data_dict: dict, post_keys: dict, force_update=False): + name_or_id = data_dict.get(post_keys.get("IMDB_ID"), "") or data_dict.get( + post_keys.get("VIDEO_TITLE"), "" + ) + imdb_metadata = lookup_video_from_imdb(name_or_id) + # skatevideosite_metadata = lookup_video_from_skatevideosite(name_or_id) + # youtube_metadata = lookup_vide_from_youtube(name_or_id) - if not imdb_dict: + video_dict = imdb_metadata + # video_metadata = imdb_metadata or skatevideosite_metadata or youtube_metadata + if not video_dict: + logger.info( + "No video found on imdb, skatevideosite or youtube, cannot scrobble", + extra={"name_or_id": name_or_id}, + ) return video, video_created = Video.objects.get_or_create( - imdb_id=imdb_dict.get("imdbID"), title=imdb_dict.get("title") + imdb_id=video_dict.get("imdb_id"), + title=video_dict.get("title"), ) - if video_created or force_update: - video_type = Video.VideoType.MOVIE series = None - if imdb_dict.get("kind") == "episode": - series_name = imdb_dict.get("episode of").data.get("title") + if video_dict.get("video_type") == Video.VideoType.TV_EPISODE: + + series_name = video_dict.pop("series_name") series, series_created = Series.objects.get_or_create( name=series_name ) - video_type = Video.VideoType.TV_EPISODE if series_created: series.fix_metadata() + video_dict["tv_series_id"] = series.id + + if genres := video_dict.pop("genres", None): + video.genre.add(*genres) + + if cover_url := video_dict.pop("cover_url", None): + video.scrape_cover_from_url(cover_url) - run_time_seconds = 0 - if imdb_dict.get("runtimes"): - run_time_seconds = int(imdb_dict.get("runtimes")[0]) * 60 - video_dict = { - "video_type": video_type, - "run_time_seconds": run_time_seconds, - "episode_number": imdb_dict.get("episode", None), - "season_number": imdb_dict.get("season", None), - "next_imdb_id": imdb_dict.get("next episode", None), - "tv_series_id": series.id if series else None, - } Video.objects.filter(pk=video.id).update(**video_dict) video.refresh_from_db() - - video.fix_metadata() - return video +def get_or_create_video_from_skatevideosite(title: str, force_update=True): + ... + + def get_or_create_video_from_jellyfin(jellyfin_data: dict, force_update=True): """Given a Jellyfin webhook payload as a dictionary, lookup the video or create a new one. """ - video, video_created = Video.objects.get_or_create( - imdb_id=jellyfin_data.get("Provider_imdb").replace("tt", ""), + imdb_id=jellyfin_data.get("Provider_imdb", "").replace("tt", ""), title=jellyfin_data.get("Name"), )