[scrobbling] Refactor webhook and simplify

This commit is contained in:
2024-08-12 00:03:30 -05:00
parent cc9a2a64df
commit 5e22cb3106
11 changed files with 249 additions and 301 deletions

View File

@ -45,7 +45,7 @@ def test_scrobble_mopidy_track(
assert scrobble.media_obj.title == "Same in the End"
@pytest.mark.skip(reason="API is unstable")
@pytest.mark.skip(reason="Allmusic API is unstable")
@pytest.mark.django_db
def test_scrobble_mopidy_same_track_different_album(
client,

View File

@ -29,12 +29,6 @@ class LifeEvent(ScrobblableMixin):
def find_or_create(cls, title: str) -> "LifeEvent":
return cls.objects.filter(title=title).first()
def scrobble_for_user(self, user_id):
Scrobble = apps.get_model("scrobbles", "Scrobble")
return Scrobble.objects.create(
user_id=user_id, life_event=self, timestamp=pendulum.now()
)
def scrobbles(self, user_id):
Scrobble = apps.get_model("scrobbles", "Scrobble")
return Scrobble.objects.filter(

View File

@ -7,7 +7,7 @@ VARIOUS_ARTIST_DICT = {
JELLYFIN_POST_KEYS = {
"ITEM_TYPE": "ItemType",
"RUN_TIME": "RunTime",
"TITLE": "Name",
"TRACK_TITLE": "Name",
"TIMESTAMP": "UtcTimestamp",
"YEAR": "Year",
"PLAYBACK_POSITION_TICKS": "PlaybackPositionTicks",
@ -18,4 +18,21 @@ JELLYFIN_POST_KEYS = {
"TRACK_MB_ID": "Provider_musicbrainztrack",
"ALBUM_NAME": "Album",
"ARTIST_NAME": "Artist",
"STATUS": "Status",
}
MOPIDY_POST_KEYS = {
"ITEM_TYPE": None,
"RUN_TIME": "run_time",
"TRACK_TITLE": "name",
"TIMESTAMP": None,
"YEAR": None,
"PLAYBACK_POSITION_TICKS": "playback_time_ticks",
"ARTIST_MB_ID": "muscibrainz_artist_id",
"ALBUM_MB_ID": "musicbrainz_album_id",
"RELEASEGROUP_MB_ID": None,
"TRACK_MB_ID": "musicbrainz_track_id",
"ALBUM_NAME": "album",
"ARTIST_NAME": "artist",
"STATUS": "status",
}

View File

@ -104,22 +104,37 @@ def lookup_artist_from_mb(artist_name: str) -> dict:
def lookup_track_from_mb(
track_name: str, artist_mbid: str, album_mbid: str
) -> str:
track_name: str, artist_mb_id: str, album_mb_id: str
) -> dict:
logger.info(
"[lookup_track_from_mb] called",
extra={
"track_name": track_name,
"artist_mb_id": artist_mb_id,
"album_mb_id": album_mb_id,
},
)
musicbrainzngs.set_useragent("vrobbler", "0.3.0")
try:
top_result = musicbrainzngs.search_recordings(
query=track_name, artist=artist_mbid, release=album_mbid
)["recording-list"][0]
results = musicbrainzngs.search_recordings(
query=track_name, artist=artist_mb_id, release=album_mb_id
)["recording-list"]
logger.info(
"[lookup_track_from_mb] musicbrainz recordings search results",
extra={"results": results},
)
top_result = results[0]
except IndexError:
return ""
logger.error("[lookup_track_from_mb] no results found")
return {}
score = int(top_result.get("ext:score"))
if score < 85:
logger.debug(
"Track lookup score below 85 threshold",
extra={"result": top_result},
logger.info(
"[lookup_track_from_mb] no results above 85% certainty ",
extra={"results": results},
)
return ""
return {}
return top_result

View File

@ -8,6 +8,7 @@ from music.musicbrainz import (
lookup_track_from_mb,
)
from music.constants import VARIOUS_ARTIST_DICT
from scrobbles.utils import convert_to_seconds
logger = logging.getLogger(__name__)
@ -89,36 +90,49 @@ def get_or_create_album(
return album
def get_or_create_track(
title: str,
artist: Artist,
album: Album = None,
mbid: str = None,
run_time_seconds=None,
) -> Track:
def get_or_create_track(post_data: dict, post_keys: dict):
artist_name = post_data.get(post_keys.get("ARTIST_NAME"), "")
artist_mb_id = post_data.get(post_keys.get("ARTIST_MB_ID"), "")
album_title = post_data.get(post_keys.get("ALBUM_NAME"), "")
album_mb_id = post_data.get(post_keys.get("ALBUM_MB_ID"), "")
track_run_time_seconds = convert_to_seconds(
post_data.get(post_keys.get("RUN_TIME"), 0)
)
track_title = post_data.get(post_keys.get("TRACK_TITLE"), "")
track_mb_id = post_data.get(post_keys.get("TRACK_MB_ID"), "")
artist = get_or_create_artist(
artist_name,
mbid=artist_mb_id,
)
album = get_or_create_album(
album_title,
artist=artist,
mbid=album_mb_id,
)
track = None
if not mbid and album:
if not track_mb_id and album:
try:
mbid = lookup_track_from_mb(
title,
track_mb_id = lookup_track_from_mb(
track_title,
artist.musicbrainz_id,
album.musicbrainz_id,
).get("id", 0)
except TypeError:
pass
if mbid:
track = Track.objects.filter(musicbrainz_id=mbid).first()
if track_mb_id:
track = Track.objects.filter(musicbrainz_id=track_mb_id).first()
if not track:
track = Track.objects.create(
title=title,
title=track_title,
artist=artist,
album=album,
musicbrainz_id=mbid,
run_time_seconds=run_time_seconds,
musicbrainz_id=track_mb_id,
run_time_seconds=track_run_time_seconds,
)
return track

View File

@ -0,0 +1,79 @@
import logging
import os
from urllib.parse import unquote
from dateutil.parser import ParserError, parse
from podcasts.models import PodcastEpisode
logger = logging.getLogger(__name__)
# TODO This should be configurable in settings or per deploy
PODCAST_DATE_FORMAT = "YYYY-MM-DD"
def parse_mopidy_uri(uri: str) -> dict:
logger.debug(f"Parsing URI: {uri}")
parsed_uri = os.path.splitext(unquote(uri))[0].split("/")
episode_str = parsed_uri[-1]
podcast_name = parsed_uri[-2].strip()
episode_num = None
episode_num_pad = 0
try:
# Without episode numbers the date will lead
pub_date = parse(episode_str[0:10])
except ParserError:
episode_num = int(episode_str.split("-")[0])
episode_num_pad = len(str(episode_num)) + 1
try:
# Beacuse we have epsiode numbers on
pub_date = parse(
episode_str[
episode_num_pad : len(PODCAST_DATE_FORMAT)
+ episode_num_pad
]
)
except ParserError:
pub_date = ""
gap_to_strip = 0
if pub_date:
gap_to_strip += len(PODCAST_DATE_FORMAT)
if episode_num:
gap_to_strip += episode_num_pad
episode_name = episode_str[gap_to_strip:].replace("-", " ").strip()
return {
"episode_filename": episode_name,
"episode_num": episode_num,
"podcast_name": podcast_name,
"pub_date": pub_date,
}
def get_or_create_podcast(post_data: dict):
mopidy_uri = post_data.get("mopidy_uri", "")
parsed_data = parse_mopidy_uri(mopidy_uri)
producer_dict = {"name": post_data.get("artist")}
podcast_name = post_data.get("album")
if not podcast_name:
podcast_name = parsed_data.get("podcast_name")
podcast_dict = {"name": podcast_name}
episode_name = parsed_data.get("episode_filename")
episode_dict = {
"title": episode_name,
"run_time_seconds": post_data.get("run_time"),
"number": parsed_data.get("episode_num"),
"pub_date": parsed_data.get("pub_date"),
"mopidy_uri": mopidy_uri,
}
return PodcastEpisode.find_or_create(
podcast_dict, producer_dict, episode_dict
)

View File

@ -47,21 +47,27 @@ class ScrobblableMixin(TimeStampedModel):
class Meta:
abstract = True
def basic_scrobble_data(self, user_id) -> dict:
return {
def scrobble_for_user(
self,
user_id,
source: str = "Vrobbler",
playback_position_seconds: int = 0,
status: str = "started",
):
Scrobble = apps.get_model("scrobbles", "Scrobble")
scrobble_data = {
"user_id": user_id,
"timestamp": timezone.now(),
"playback_position_seconds": 0,
"source": "Vrobbler",
"source": source,
"playback_position_seconds": playback_position_seconds,
"status": status,
}
def scrobble_for_user(self, user_id):
Scrobble = apps.get_model("scrobbles", "Scrobble")
scrobble_data = self.basic_scrobble_data(user_id)
logger.info(
"[scrobble_for_user] called",
extra={
"webpage_id": self.id,
"id": self.id,
"media_type": self.__class__,
"user_id": user_id,
"scrobble_data": scrobble_data,
"media_type": Scrobble.MediaType.WEBPAGE,
@ -90,12 +96,6 @@ class ScrobblableMixin(TimeStampedModel):
def find_or_create(cls) -> None:
logger.warn("find_or_create() not implemented yet")
def scrobble(self, user_id, **kwargs):
"""Given a user ID and a dictionary of data, attempts to scrobble it"""
from scrobbles.models import Scrobble
Scrobble.create_or_update(self, user_id, **kwargs)
class LongPlayScrobblableMixin(ScrobblableMixin):
class Meta:

View File

@ -648,7 +648,10 @@ class Scrobble(TimeStampedModel):
@property
def is_stale(self) -> bool:
"""Mark scrobble as stale if it's been more than an hour since it was updated"""
"""Mark scrobble as stale if it's been more than an hour since it was updated
Effectively, this allows 'resuming' a video scrobble within an hour of starting it.
"""
is_stale = False
now = timezone.now()
seconds_since_last_update = (now - self.modified).seconds
@ -911,7 +914,7 @@ class Scrobble(TimeStampedModel):
# GeoLocations are a special case scrobble
if mtype == cls.MediaType.GEO_LOCATION:
logger.warn(
f"[scrobbling] use create_or_update_location for GeoLocations"
f"[create_or_update] geoloc requires create_or_update_location"
)
scrobble = cls.create_or_update_location(
media, scrobble_data, user_id
@ -919,22 +922,16 @@ class Scrobble(TimeStampedModel):
return scrobble
logger.info(
f"[scrobbling] check for existing scrobble to update ",
f"[create_or_update] check for existing scrobble to update ",
extra={
"scrobble_id": scrobble.id if scrobble else None,
"media_type": mtype,
"media_id": media.id,
"scrobble_data": scrobble_data,
"percent_played": scrobble.percent_played if scrobble else 0,
"can_be_updated": scrobble.can_be_updated
if scrobble
else False,
},
)
scrobble_data["playback_status"] = scrobble_data.pop(
"mopidy_status", scrobble_data.pop("jellyfin_status", None)
)
scrobble_data["playback_status"] = scrobble_data.pop("status", None)
# If it's marked as stopped, send it through our update mechanism, which will complete it
if scrobble and (
scrobble.can_be_updated
@ -1069,7 +1066,7 @@ class Scrobble(TimeStampedModel):
playback_status = scrobble_data.pop("playback_status", None)
logger.info(
"[scrobbling] update called",
"[update] called",
extra={
"scrobble_id": self.id,
"scrobble_data": scrobble_data,
@ -1101,16 +1098,6 @@ class Scrobble(TimeStampedModel):
update_fields.append(key)
self.save(update_fields=update_fields)
logger.info(
"[scrobbling] update finished",
extra={
"scrobble_id": self.id,
"scrobble_data": scrobble_data,
"playback_status": playback_status,
"media_type": self.media_type,
},
)
return self
@classmethod

View File

@ -1,25 +1,21 @@
import json
import logging
from typing import Optional
import pendulum
from boardgames.bgg import lookup_boardgame_from_bgg
from boardgames.models import BoardGame
from books.models import Book
from books.openlibrary import lookup_book_from_openlibrary
from dateutil.parser import parse
from django.utils import timezone
from locations.constants import LOCATION_PROVIDERS
from locations.models import GeoLocation
from music.constants import JELLYFIN_POST_KEYS
from music.constants import JELLYFIN_POST_KEYS, MOPIDY_POST_KEYS
from music.models import Track
from music.utils import (
get_or_create_album,
get_or_create_artist,
get_or_create_track,
)
from podcasts.models import PodcastEpisode
from music.utils import get_or_create_track
from podcasts.utils import get_or_create_podcast
from scrobbles.constants import JELLYFIN_AUDIO_ITEM_TYPES
from scrobbles.models import Scrobble
from scrobbles.utils import convert_to_seconds, parse_mopidy_uri
from scrobbles.utils import convert_to_seconds
from sports.models import SportEvent
from sports.thesportsdb import lookup_event_from_thesportsdb
from videogames.howlongtobeat import lookup_game_from_hltb
@ -30,180 +26,88 @@ from webpages.models import WebPage
logger = logging.getLogger(__name__)
def mopidy_scrobble_podcast(
data_dict: dict, user_id: Optional[int]
) -> Scrobble:
mopidy_uri = data_dict.get("mopidy_uri", "")
parsed_data = parse_mopidy_uri(mopidy_uri)
producer_dict = {"name": data_dict.get("artist")}
podcast_name = data_dict.get("album")
if not podcast_name:
podcast_name = parsed_data.get("podcast_name")
podcast_dict = {"name": podcast_name}
episode_name = parsed_data.get("episode_filename")
episode_dict = {
"title": episode_name,
"run_time_seconds": data_dict.get("run_time"),
"number": parsed_data.get("episode_num"),
"pub_date": parsed_data.get("pub_date"),
"mopidy_uri": mopidy_uri,
}
episode = PodcastEpisode.find_or_create(
podcast_dict, producer_dict, episode_dict
)
# Now we run off a scrobble
mopidy_data = {
"user_id": user_id,
"timestamp": timezone.now(),
"playback_position_seconds": data_dict.get("playback_time_ticks"),
"source": "Mopidy",
"mopidy_status": data_dict.get("status"),
}
def mopidy_scrobble_media(post_data: dict, user_id: int) -> Scrobble:
media_type = Scrobble.MediaType.TRACK
if "podcast" in post_data.get("mopidy_uri", ""):
media_type = Scrobble.MediaType.PODCAST_EPISODE
logger.info(
"[scrobblers] webhook mopidy scrobble request received",
extra={
"episode_id": episode.id if episode else None,
"user_id": user_id,
"scrobble_dict": mopidy_data,
"media_type": Scrobble.MediaType.PODCAST_EPISODE,
"post_data": post_data,
"media_type": media_type,
},
)
scrobble = None
if episode:
scrobble = Scrobble.create_or_update(episode, user_id, mopidy_data)
return scrobble
def mopidy_scrobble_track(
data_dict: dict, user_id: Optional[int]
) -> Optional[Scrobble]:
artist = get_or_create_artist(
data_dict.get("artist"),
mbid=data_dict.get("musicbrainz_artist_id", None),
)
album = get_or_create_album(
data_dict.get("album"),
artist=artist,
mbid=data_dict.get("musicbrainz_album_id"),
)
track = get_or_create_track(
title=data_dict.get("name"),
mbid=data_dict.get("musicbrainz_track_id"),
artist=artist,
album=album,
run_time_seconds=data_dict.get("run_time"),
)
if media_type == Scrobble.MediaType.PODCAST_EPISODE:
media_obj = get_or_create_podcast(post_data)
else:
media_obj = get_or_create_track(post_data, MOPIDY_POST_KEYS)
# Now we run off a scrobble
playback_seconds = data_dict.get("playback_time_ticks") / 1000
mopidy_data = {
"user_id": user_id,
"timestamp": timezone.now(),
"playback_position_seconds": playback_seconds,
"source": "Mopidy",
"mopidy_status": data_dict.get("status"),
}
playback_seconds = post_data.get("playback_time_ticks", 1) / 1000
playback_status = post_data.get(MOPIDY_POST_KEYS.get("STATUS"), "")
return media_obj.scrobble_for_user(
user_id,
source="Mopidy",
playback_position_seconds=playback_seconds,
status=playback_status,
)
def jellyfin_scrobble_media(
post_data: dict, user_id: int
) -> Optional[Scrobble]:
media_type = Scrobble.MediaType.VIDEO
if post_data.pop("ItemType", "") in JELLYFIN_AUDIO_ITEM_TYPES:
media_type = Scrobble.MediaType.TRACK
logger.info(
"[scrobblers] webhook mopidy scrobble request received",
"[jellyfin_scrobble_track] called",
extra={
"track_id": track.id,
"user_id": user_id,
"scrobble_dict": mopidy_data,
"media_type": Scrobble.MediaType.TRACK,
"post_data": post_data,
"media_type": media_type,
},
)
scrobble = Scrobble.create_or_update(track, user_id, mopidy_data)
return scrobble
def build_scrobble_dict(data_dict: dict, user_id: int) -> dict:
jellyfin_status = "resumed"
if data_dict.get("IsPaused"):
jellyfin_status = "paused"
elif data_dict.get("NotificationType") == "PlaybackStop":
jellyfin_status = "stopped"
playback_seconds = convert_to_seconds(
data_dict.get("PlaybackPosition", "")
)
return {
"user_id": user_id,
"timestamp": parse(data_dict.get("UtcTimestamp")),
"playback_position_seconds": playback_seconds,
"source": data_dict.get("ClientName", "Vrobbler"),
"log": {"media_source_id": data_dict.get("MediaSourceId")},
"jellyfin_status": jellyfin_status,
}
def jellyfin_scrobble_track(
data_dict: dict, user_id: Optional[int]
) -> Optional[Scrobble]:
null_position_on_progress = (
data_dict.get("PlaybackPosition") == "00:00:00"
and data_dict.get("NotificationType") == "PlaybackProgress"
post_data.get("PlaybackPosition") == "00:00:00"
and post_data.get("NotificationType") == "PlaybackProgress"
)
playback_status = "resumed"
if post_data.get("IsPaused"):
playback_status = "paused"
elif post_data.get("NotificationType") == "PlaybackStop":
playback_status = "stopped"
# Jellyfin has some race conditions with it's webhooks, these hacks fix some of them
if null_position_on_progress:
logger.error("No playback position tick from Jellyfin, aborting")
logger.info(
"[jellyfin_scrobble_track] no playback position tick, aborting",
extra={"post_data": post_data},
)
return
artist = get_or_create_artist(
data_dict.get(JELLYFIN_POST_KEYS["ARTIST_NAME"]),
mbid=data_dict.get(JELLYFIN_POST_KEYS["ARTIST_MB_ID"]),
)
album = get_or_create_album(
data_dict.get(JELLYFIN_POST_KEYS["ALBUM_NAME"]),
artist=artist,
mbid=data_dict.get(JELLYFIN_POST_KEYS["ALBUM_MB_ID"]),
playback_position_seconds = convert_to_seconds(
post_data.get(JELLYFIN_POST_KEYS.get("RUN_TIME"), 0)
)
run_time = convert_to_seconds(
data_dict.get(JELLYFIN_POST_KEYS["RUN_TIME"])
if media_type == Scrobble.MediaType.VIDEO:
media_obj = Video.find_or_create(post_data)
else:
media_obj = get_or_create_track(
post_data, post_keys=JELLYFIN_POST_KEYS
)
return media_obj.scrobble_for_user_id(
user_id,
playback_position_seconds=playback_position_seconds,
status=playback_status,
)
track = get_or_create_track(
title=data_dict.get("Name"),
artist=artist,
album=album,
run_time_seconds=run_time,
)
scrobble_dict = build_scrobble_dict(data_dict, user_id)
# A hack to make Jellyfin work more like Mopidy for music tracks
scrobble_dict["playback_position_seconds"] = 0
return Scrobble.create_or_update(track, user_id, scrobble_dict)
def jellyfin_scrobble_video(data_dict: dict, user_id: Optional[int]):
video = Video.find_or_create(data_dict)
scrobble_dict = build_scrobble_dict(data_dict, user_id)
logger.info(
"[scrobblers] webhook video scrobble request received",
extra={
"video_id": video.id,
"user_id": user_id,
"scrobble_dict": scrobble_dict,
"media_type": Scrobble.MediaType.VIDEO,
},
)
return Scrobble.create_or_update(video, user_id, scrobble_dict)
def manual_scrobble_video(imdb_id: str, user_id: int):

View File

@ -1,11 +1,8 @@
import logging
import os
import re
from datetime import datetime, timedelta, tzinfo
from urllib.parse import unquote
import pytz
from dateutil.parser import ParserError, parse
from django.apps import apps
from django.contrib.auth import get_user_model
from django.db import models
@ -19,9 +16,6 @@ logger = logging.getLogger(__name__)
User = get_user_model()
PODCAST_DATE_FORMAT = "YYYY-MM-DD"
def timestamp_user_tz_to_utc(timestamp: int, user_tz: tzinfo) -> datetime:
return user_tz.localize(datetime.utcfromtimestamp(timestamp)).astimezone(
pytz.utc
@ -45,49 +39,6 @@ def convert_to_seconds(run_time: str) -> int:
return run_time_int
def parse_mopidy_uri(uri: str) -> dict:
logger.debug(f"Parsing URI: {uri}")
parsed_uri = os.path.splitext(unquote(uri))[0].split("/")
episode_str = parsed_uri[-1]
podcast_name = parsed_uri[-2].strip()
episode_num = None
episode_num_pad = 0
try:
# Without episode numbers the date will lead
pub_date = parse(episode_str[0:10])
except ParserError:
episode_num = int(episode_str.split("-")[0])
episode_num_pad = len(str(episode_num)) + 1
try:
# Beacuse we have epsiode numbers on
pub_date = parse(
episode_str[
episode_num_pad : len(PODCAST_DATE_FORMAT)
+ episode_num_pad
]
)
except ParserError:
pub_date = ""
gap_to_strip = 0
if pub_date:
gap_to_strip += len(PODCAST_DATE_FORMAT)
if episode_num:
gap_to_strip += episode_num_pad
episode_name = episode_str[gap_to_strip:].replace("-", " ").strip()
return {
"episode_filename": episode_name,
"episode_num": episode_num,
"podcast_name": podcast_name,
"pub_date": pub_date,
}
def get_scrobbles_for_media(media_obj, user: User) -> models.QuerySet:
Scrobble = apps.get_model(app_label="scrobbles", model_name="Scrobble")

View File

@ -320,29 +320,24 @@ def lastfm_import(request):
@permission_classes([IsAuthenticated])
@api_view(["POST"])
def jellyfin_webhook(request):
data_dict = request.data
post_data = request.data
logger.info(
"[jellyfin_webhook] called",
extra={"post_data": post_data},
)
# For making things easier to build new input processors
if getattr(settings, "DUMP_REQUEST_DATA", False):
json_data = json.dumps(data_dict, indent=4)
logger.info(f"{json_data}")
in_progress = data_dict.get("NotificationType", "") == "PlaybackProgress"
is_music = data_dict.get("ItemType", "") == "Audio"
in_progress = post_data.get("NotificationType", "") == "PlaybackProgress"
is_music = post_data.get("ItemType", "") == "Audio"
# Disregard progress updates
if in_progress and is_music:
logger.debug("Disregarding playback update from Jellyfin")
logger.info(
"[jellyfin_webhook] ignoring update of music in progress",
extra={"post_data": post_data},
)
return Response({}, status=status.HTTP_304_NOT_MODIFIED)
scrobble = None
media_type = data_dict.get("ItemType", "")
if media_type in JELLYFIN_AUDIO_ITEM_TYPES:
scrobble = jellyfin_scrobble_track(data_dict, request.user.id)
if media_type in JELLYFIN_VIDEO_ITEM_TYPES:
scrobble = jellyfin_scrobble_video(data_dict, request.user.id)
scrobble = jellyfin_scrobble_media(post_data, request.user.id)
if not scrobble:
return Response({}, status=status.HTTP_400_BAD_REQUEST)
@ -359,15 +354,7 @@ def mopidy_webhook(request):
except TypeError:
data_dict = request.data
# For making things easier to build new input processors
if getattr(settings, "DUMP_REQUEST_DATA", False):
json_data = json.dumps(data_dict, indent=4)
logger.debug(f"{json_data}")
if "podcast" in data_dict.get("mopidy_uri"):
scrobble = mopidy_scrobble_podcast(data_dict, request.user.id)
else:
scrobble = mopidy_scrobble_track(data_dict, request.user.id)
scrobble = mopidy_scrobble_media(data_dict, request.user.id)
if not scrobble:
return Response({}, status=status.HTTP_400_BAD_REQUEST)