From b0e9f13e11521e694881c49720f7febc82d3e22a Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Sun, 6 Apr 2025 22:28:32 -0400 Subject: [PATCH] [music] Attempts to fix bad lookups from LastFM and Jellyfin Broader issue was creating tracks without albums that were duplicates of existing tracks because sometimes Jellyfin and LastFM do not have albums sent with them. --- vrobbler/apps/music/lastfm.py | 22 +- .../music/migrations/0025_artist_alt_names.py | 18 ++ .../music/migrations/0026_album_alt_names.py | 18 ++ vrobbler/apps/music/models.py | 255 +++++++++++++++--- vrobbler/apps/music/utils.py | 54 ++-- vrobbler/apps/scrobbles/mixins.py | 6 +- vrobbler/apps/scrobbles/scrobblers.py | 16 +- vrobbler/apps/scrobbles/tsv.py | 33 +-- 8 files changed, 323 insertions(+), 99 deletions(-) create mode 100644 vrobbler/apps/music/migrations/0025_artist_alt_names.py create mode 100644 vrobbler/apps/music/migrations/0026_album_alt_names.py diff --git a/vrobbler/apps/music/lastfm.py b/vrobbler/apps/music/lastfm.py index 3cacb8a..30dc07b 100644 --- a/vrobbler/apps/music/lastfm.py +++ b/vrobbler/apps/music/lastfm.py @@ -1,14 +1,10 @@ import logging -import time -from datetime import datetime, timedelta, UTC +from datetime import datetime, timedelta import pylast import pytz from django.conf import settings -from django.utils import timezone -from music.utils import ( - get_or_create_track, -) +from music.models import Track logger = logging.getLogger(__name__) @@ -47,14 +43,10 @@ class LastFM: lastfm_scrobbles = self.get_last_scrobbles(time_from=last_processed) for lfm_scrobble in lastfm_scrobbles: - track = get_or_create_track( - lfm_scrobble, - { - "TRACK_TITLE": "title", - "ARTIST_NAME": "artist", - "ALBUM_NAME": "album", - "RUN_TIME": "run_time_seconds", - }, + track = Track.find_or_create( + title=lfm_scrobble.get("title"), + artist_name=lfm_scrobble.get("artist"), + album_name=lfm_scrobble.get("album"), ) timezone = settings.TIME_ZONE @@ -149,7 +141,7 @@ class LastFM: continue # TODO figure out if this will actually work - #timestamp = datetime.fromtimestamp(int(scrobble.timestamp), UTC) + # timestamp = datetime.fromtimestamp(int(scrobble.timestamp), UTC) timestamp = datetime.utcfromtimestamp( int(scrobble.timestamp) ).replace(tzinfo=pytz.utc) diff --git a/vrobbler/apps/music/migrations/0025_artist_alt_names.py b/vrobbler/apps/music/migrations/0025_artist_alt_names.py new file mode 100644 index 0000000..d148c71 --- /dev/null +++ b/vrobbler/apps/music/migrations/0025_artist_alt_names.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.19 on 2025-04-07 00:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("music", "0024_alter_track_run_time_seconds"), + ] + + operations = [ + migrations.AddField( + model_name="artist", + name="alt_names", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/vrobbler/apps/music/migrations/0026_album_alt_names.py b/vrobbler/apps/music/migrations/0026_album_alt_names.py new file mode 100644 index 0000000..9db0661 --- /dev/null +++ b/vrobbler/apps/music/migrations/0026_album_alt_names.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.19 on 2025-04-07 00:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("music", "0025_artist_alt_names"), + ] + + operations = [ + migrations.AddField( + model_name="album", + name="alt_names", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/vrobbler/apps/music/models.py b/vrobbler/apps/music/models.py index ce7a0fd..6ceaf56 100644 --- a/vrobbler/apps/music/models.py +++ b/vrobbler/apps/music/models.py @@ -1,13 +1,11 @@ import logging -from tempfile import NamedTemporaryFile from typing import Dict, Optional -from urllib.request import urlopen from uuid import uuid4 import musicbrainzngs import requests from django.conf import settings -from django.core.files.base import ContentFile, File +from django.core.files.base import ContentFile from django.db import models from django.urls import reverse from django.utils.translation import gettext_lazy as _ @@ -16,6 +14,7 @@ from imagekit.models import ImageSpecField from imagekit.processors import ResizeToFit from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic from music.bandcamp import get_bandcamp_slug +from music.musicbrainz import lookup_album_dict_from_mb, lookup_track_from_mb from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin @@ -24,6 +23,16 @@ BNULL = {"blank": True, "null": True} class Artist(TimeStampedModel): + """Represents a music artist. + + # Lookup or create by title alone + >>> Artist.find_or_create(name="Bon Iver") + + # Lookup or create by MB id alone + >>> Artist.find_or_create(musicbrainz_id="0307edfc-437c-4b48-8700-80680e66a228") + + """ + uuid = models.UUIDField(default=uuid4, editable=False, **BNULL) name = models.CharField(max_length=255) biography = models.TextField(**BNULL) @@ -46,6 +55,7 @@ class Artist(TimeStampedModel): format="JPEG", options={"quality": 75}, ) + alt_names = models.TextField(**BNULL) class Meta: unique_together = [["name", "musicbrainz_id"]] @@ -62,8 +72,10 @@ class Artist(TimeStampedModel): return "" @property - def mb_link(self): - return f"https://musicbrainz.org/artist/{self.musicbrainz_id}" + def mb_link(self) -> str: + if self.musicbrainz_id: + return f"https://musicbrainz.org/artist/{self.musicbrainz_id}" + return "" @property def allmusic_link(self): @@ -104,7 +116,9 @@ class Artist(TimeStampedModel): if not self.allmusic_id or force: slug = get_allmusic_slug(self.name) if not slug: - logger.info(f"No allmsuic link for {self}") + logger.info( + "No allmusic link found", extra={"track_id": self.id} + ) return self.allmusic_id = slug self.save(update_fields=["allmusic_id"]) @@ -113,7 +127,9 @@ class Artist(TimeStampedModel): if not self.bandcamp_id or force: slug = get_bandcamp_slug(self.name) if not slug: - logger.info(f"No bandcamp link for {self}") + logger.info( + "No bandcamp link found", extra={"track_id": self.id} + ) return self.bandcamp_id = slug self.save(update_fields=["bandcamp_id"]) @@ -153,6 +169,61 @@ class Artist(TimeStampedModel): artist = self.name.lower() return f"https://bandcamp.com/search?q={artist}&item_type=b" + @classmethod + def find_or_create(cls, name: str, musicbrainz_id: str = "") -> "Artist": + from music.musicbrainz import lookup_artist_from_mb + from music.utils import clean_artist_name + + if not name: + raise Exception("Must have name to lookup artist") + + artist = None + name = clean_artist_name(name) + + # Check for name/mbid combo, just mbid and then just name + if musicbrainz_id: + artist = cls.objects.filter( + name=name, musicbrainz_id=musicbrainz_id + ).first() + if not artist: + artist = cls.objects.filter(musicbrainz_id=musicbrainz_id).first() + if not artist: + artist = cls.objects.filter( + models.Q(name=name) | models.Q(alt_names__icontains=name) + ).first() + + # Does not exist, look it up from Musicbrainz + if not artist: + alt_name = None + try: + artist_dict = lookup_artist_from_mb(name) + musicbrainz_id = musicbrainz_id or artist_dict.get("id", "") + if name != artist_dict.get("name", ""): + alt_name = name + name = artist_dict.get("name", "") + except ValueError: + pass + + if musicbrainz_id: + artist = cls.objects.filter( + musicbrainz_id=musicbrainz_id + ).first() + if artist and alt_name: + if not artist.alt_names: + artist.alt_names = alt_name + else: + artist.alt_names += f"\\{alt_name}" + artist.save(update_fields=["alt_names"]) + + if not artist: + artist = cls.objects.create( + name=name, musicbrainz_id=musicbrainz_id, alt_names=alt_name + ) + # TODO maybe this should be spun off into an async task? + artist.fix_metadata() + + return artist + class Album(TimeStampedModel): uuid = models.UUIDField(default=uuid4, editable=False, **BNULL) @@ -196,9 +267,10 @@ class Album(TimeStampedModel): wikipedia_slug = models.CharField(max_length=255, **BNULL) discogs_id = models.CharField(max_length=255, **BNULL) wikidata_id = models.CharField(max_length=255, **BNULL) + alt_names = models.TextField(**BNULL) - def __str__(self): - return self.name + def __str__(self) -> str: + return "{} by {}".format(self.name, self.album_artist) def get_absolute_url(self): return reverse("music:album_detail", kwargs={"slug": self.uuid}) @@ -402,6 +474,69 @@ class Album(TimeStampedModel): album = self.name.lower() return f"https://bandcamp.com/search?q={album} {artist}&item_type=a" + @classmethod + def find_or_create( + cls, name: str, artist_name: str, musicbrainz_id: str = "" + ) -> "Album": + if not name or not artist_name: + raise Exception( + "Must have at least name and artist name to lookup album" + ) + + album = None + if musicbrainz_id: + album = cls.objects.filter( + musicbrainz_id=musicbrainz_id, + name=name, + album_artist__name=artist_name, + ).first() + if not album and musicbrainz_id: + album = cls.objects.filter( + musicbrainz_id=musicbrainz_id, + ).first() + if not album: + album = cls.objects.filter( + models.Q(name=name) | models.Q(alt_names__icontains=name), + album_artist__name=artist_name, + ).first() + + if not album: + alt_name = None + try: + album_dict = lookup_album_dict_from_mb( + name, artist_name=artist_name + ) + musicbrainz_id = musicbrainz_id or album_dict.get("mb_id", "") + found_name = album_dict.get("title", "") + if found_name and name != found_name: + alt_name = name + name = found_name + except ValueError: + pass + + if musicbrainz_id: + album = cls.objects.filter( + musicbrainz_id=musicbrainz_id + ).first() + if album and alt_name: + if not album.alt_names: + album.alt_names = alt_name + else: + album.alt_names += f"\\{alt_name}" + album.save(update_fields=["alt_names"]) + if not album: + artist = Artist.find_or_create(name=artist_name) + album = cls.objects.create( + name=name, + album_artist=artist, + musicbrainz_id=musicbrainz_id, + alt_names=alt_name, + ) + # TODO maybe do this in a separate process? + album.fix_metadata() + + return album + class Track(ScrobblableMixin): COMPLETION_PERCENT = getattr(settings, "MUSIC_COMPLETION_PERCENT", 100) @@ -425,8 +560,8 @@ class Track(ScrobblableMixin): return reverse("music:track_detail", kwargs={"slug": self.uuid}) @property - def subtitle(self): - return self.artist + def subtitle(self) -> str: + return str(self.artist) @property def strings(self) -> ScrobblableConstants: @@ -451,31 +586,85 @@ class Track(ScrobblableMixin): @classmethod def find_or_create( - cls, artist_dict: Dict, album_dict: Dict, track_dict: Dict - ) -> Optional["Track"]: - """Given a data dict from Jellyfin, does the heavy lifting of looking up - the video and, if need, TV Series, creating both if they don't yet - exist. + cls, + title: str = "", + musicbrainz_id: str = "", + album_name: str = "", + artist_name: str = "", + enrich: bool = True, + run_time_seconds: Optional[int] = None, + ) -> "Track": + # TODO we can use Q to build queries here based on whether we have mbid and album name + track = None + # Full look up with MB ID + if album_name: + track = cls.objects.filter( + musicbrainz_id=musicbrainz_id, + title=title, + artist__name=artist_name, + album__name=album_name, + ).first() + # Full look up without album + if not track: + track = cls.objects.filter( + musicbrainz_id=musicbrainz_id, + title=title, + artist__name=artist_name, + ).first() - """ - if not artist_dict.get("name") or not artist_dict.get( - "musicbrainz_id" - ): - logger.warning( - f"No artist or artist musicbrainz ID found in message from source, not scrobbling" - ) - return + # Full look up without MB ID + if not track: + track = cls.objects.filter( + title=title, + artist__name=artist_name, + album__name=album_name, + ).first() + # Base look up without MB ID or album + if not track: + track = cls.objects.filter( + title=title, + artist__name=artist_name, + ).first() - artist, artist_created = Artist.objects.get_or_create(**artist_dict) - album, album_created = Album.objects.get_or_create(**album_dict) + if not track and enrich: + track_dict = lookup_track_from_mb(title, artist_name, album_name) + musicbrainz_id = musicbrainz_id or track_dict.get("id", "") + # TODO This only works some of the time + # try: + # album_name = album_name or track_dict.get("release-list")[ + # 0 + # ].get("title", "") + # except IndexError: + # pass + if not run_time_seconds: + run_time_seconds = int( + int(track_dict.get("length", 900000)) / 1000 + ) + if title != track_dict.get("name", "") and track_dict.get( + "name", False + ): - album.fix_metadata() - if not album.cover_image: - album.fetch_artwork() + title = track_dict.get("name", "") - track_dict["album_id"] = getattr(album, "id", None) - track_dict["artist_id"] = artist.id - - track, created = cls.objects.get_or_create(**track_dict) + if musicbrainz_id: + track = cls.objects.filter( + musicbrainz_id=musicbrainz_id + ).first() + if not track: + artist = Artist.find_or_create(name=artist_name) + album = None + if album_name: + album = Album.find_or_create( + name=album_name, artist_name=artist_name + ) + track = cls.objects.create( + title=title, + album=album, + musicbrainz_id=musicbrainz_id, + artist=artist, + run_time_seconds=run_time_seconds, + ) + # TODO maybe do this in a separate process? + track.fix_metadata() return track diff --git a/vrobbler/apps/music/utils.py b/vrobbler/apps/music/utils.py index 4e6f9b9..c050c02 100644 --- a/vrobbler/apps/music/utils.py +++ b/vrobbler/apps/music/utils.py @@ -16,9 +16,8 @@ logger = logging.getLogger(__name__) from music.models import Album, Artist, Track -def get_or_create_artist(name: str, mbid: str = None) -> Artist: - artist = None - +def clean_artist_name(name: str) -> str: + """Remove featured names from artist string.""" if "feat." in name.lower(): name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip() if "featuring" in name.lower(): @@ -26,18 +25,44 @@ def get_or_create_artist(name: str, mbid: str = None) -> Artist: if "&" in name.lower(): name = re.split("&", name, flags=re.IGNORECASE)[0].strip() - artist_dict = lookup_artist_from_mb(name) - mbid = mbid or artist_dict.get("id", None) + return name - if mbid: + +# TODO These are depreacted, remove them eventually +def get_or_create_artist(name: str, mbid: str = "") -> Artist: + """Get an Artist object from the database. + + Check if an artist with this name or Musicbrainz ID already exists. + Otherwise, go lookup artist data from Musicbrainz and create one. + + """ + artist = None + name = clean_artist_name(name) + + # Check for name/mbid combo, just mbid and then just name + artist = Artist.objects.filter(name=name, mbid=mbid).first() + if not artist: artist = Artist.objects.filter(musicbrainz_id=mbid).first() + if not artist: + artist = Artist.objects.filter(name=name).first() + + # Does not exist, look it up from Musicbrainz + if not artist: + artist_dict = lookup_artist_from_mb(name) + mbid = mbid or artist_dict.get("id", "") + + if mbid: + artist = Artist.objects.filter(musicbrainz_id=mbid).first() + if not artist: artist = Artist.objects.create(name=name, musicbrainz_id=mbid) + # TODO maybe this should be spun off into an async task? artist.fix_metadata() return artist +# TODO These are depreacted, remove them eventually def get_or_create_album( name: str, artist: Artist, mbid: str = None ) -> Optional[Album]: @@ -90,6 +115,7 @@ def get_or_create_album( return album +# TODO These are depreacted, remove them eventually def get_or_create_track(post_data: dict, post_keys: dict) -> Track: try: track_run_time_seconds = int( @@ -107,16 +133,12 @@ def get_or_create_track(post_data: dict, post_keys: dict) -> Track: track_title = post_data.get(post_keys.get("TRACK_TITLE"), "") track_mb_id = post_data.get(post_keys.get("TRACK_MB_ID"), "") - artist = get_or_create_artist( - artist_name, - mbid=artist_mb_id, - ) + artist = Artist.find_or_create(artist_name, artist_mb_id) album = None - if album_mb_id: - album = get_or_create_album( - album_title, - artist=artist, - mbid=album_mb_id, + # We may get no album ID or title, in which case, skip + if album_mb_id or album_title: + album = Album.find_or_create( + album_title, str(artist.name), album_mb_id ) track = None @@ -154,7 +176,7 @@ def get_or_create_track(post_data: dict, post_keys: dict) -> Track: return track -def get_or_create_various_artists(): +def get_or_create_various_artists() -> Artist: artist = Artist.objects.filter(name="Various Artists").first() if not artist: artist = Artist.objects.create(**VARIOUS_ARTIST_DICT) diff --git a/vrobbler/apps/scrobbles/mixins.py b/vrobbler/apps/scrobbles/mixins.py index c572d3c..812e379 100644 --- a/vrobbler/apps/scrobbles/mixins.py +++ b/vrobbler/apps/scrobbles/mixins.py @@ -124,12 +124,12 @@ class ScrobblableMixin(TimeStampedModel): logger.warning("fix_metadata() not implemented yet") @classmethod - def find_or_create(cls) -> None: + def find_or_create(cls): logger.warning("find_or_create() not implemented yet") - def __str__(self): + def __str__(self) -> str: if self.title: - return self.title + return str(self.title) return str(self.uuid) diff --git a/vrobbler/apps/scrobbles/scrobblers.py b/vrobbler/apps/scrobbles/scrobblers.py index 17a3701..dc2056c 100644 --- a/vrobbler/apps/scrobbles/scrobblers.py +++ b/vrobbler/apps/scrobbles/scrobblers.py @@ -1,8 +1,7 @@ -from datetime import datetime import logging import re +from datetime import datetime from typing import Optional -from urllib.parse import parse_qs, urlparse import pendulum import pytz @@ -56,7 +55,11 @@ def mopidy_scrobble_media(post_data: dict, user_id: int) -> Scrobble: if media_type == Scrobble.MediaType.PODCAST_EPISODE: media_obj = get_or_create_podcast(post_data) else: - media_obj = get_or_create_track(post_data, MOPIDY_POST_KEYS) + media_obj = Track.find_or_create( + title=post_data.get("title", ""), + album_name=post_data.get("album", ""), + run_time_seconds=post_data.get("run_time", 900000), + ) log = {} try: @@ -109,8 +112,11 @@ def jellyfin_scrobble_media( post_data.get("Provider_imdb", "").replace("tt", "") ) else: - media_obj = get_or_create_track( - post_data, post_keys=JELLYFIN_POST_KEYS + media_obj = Track.find_or_create( + title=post_data.get("Name", ""), + album_name=post_data.get("Album", ""), + run_time_seconds=post_data.get("RunTime", 900000), + musicbrainz_id=post_data.get("Provider_musicbrainztrack", ""), ) # A hack because we don't worry about updating music ... we either finish it or we don't playback_position_seconds = 0 diff --git a/vrobbler/apps/scrobbles/tsv.py b/vrobbler/apps/scrobbles/tsv.py index 9167512..d91f18f 100644 --- a/vrobbler/apps/scrobbles/tsv.py +++ b/vrobbler/apps/scrobbles/tsv.py @@ -1,18 +1,12 @@ import codecs import csv import logging -from datetime import datetime import pytz import requests -from music.utils import ( - get_or_create_album, - get_or_create_artist, - get_or_create_track, -) +from music.models import Track from scrobbles.constants import AsTsvColumn from scrobbles.models import Scrobble -from music.constants import MOPIDY_POST_KEYS from scrobbles.utils import timestamp_user_tz_to_utc @@ -50,27 +44,12 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None): ) continue - track = get_or_create_track( - { - "title": row[AsTsvColumn["TRACK_NAME"].value], - "mbid": row[AsTsvColumn["MB_ID"].value], - "artist_name": row[AsTsvColumn["ARTIST_NAME"].value], - "album_name": row[AsTsvColumn["ALBUM_NAME"].value], - "run_time_seconds": int( - row[AsTsvColumn["RUN_TIME_SECONDS"].value] - ), - }, - { - "TRACK_MB_ID": "mbid", - "TRACK_TITLE": "track_title", - "ALBUM_NAME": "album_name", - "ARTIST_NAME": "artist_name", - "RUN_TIME": "run_time_seconds", - }, + track = Track.find_or_create( + title=row[AsTsvColumn["TRACK_NAME"].value], + musicbrainz_id=row[AsTsvColumn["MB_ID"].value], + artist_name=row[AsTsvColumn["ARTIST_NAME"].value] + album_name=row[AsTsvColumn["ALBUM_NAME"].value] ) - if not track: - logger.info(f"Skipping track {track} because not found") - continue # TODO Set all this up as constants if row[AsTsvColumn["COMPLETE"].value] == "S":