[music] Reorganize importer and fix lookups
This commit is contained in:
@ -15,6 +15,11 @@ from imagekit.processors import ResizeToFit
|
||||
from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic
|
||||
from music.bandcamp import get_bandcamp_slug
|
||||
from music.musicbrainz import (
|
||||
get_album_metadata,
|
||||
get_album_metadata_with_artist,
|
||||
get_artist_metadata_extended,
|
||||
get_recording_mbid_exact,
|
||||
get_track_metadata_with_artist,
|
||||
lookup_album_dict_from_mb,
|
||||
lookup_album_from_mb,
|
||||
lookup_track_from_mb,
|
||||
@ -177,56 +182,75 @@ class Artist(TimeStampedModel):
|
||||
|
||||
@classmethod
|
||||
def find_or_create(
|
||||
cls, name: str = "", musicbrainz_id: str = ""
|
||||
cls, name: str, album_name: str = "", track_name: str = ""
|
||||
) -> "Artist":
|
||||
"""The biggest challenge to finding artists is that the search often
|
||||
fails miserably unless you can look it up along with an album or a track name.
|
||||
|
||||
Thus, when we find or create an artist, we should always provide an optional
|
||||
album name or track name, but probably not both."""
|
||||
if album_name:
|
||||
logger.info(
|
||||
f"Looking for artist with name {name} and album {album_name}"
|
||||
)
|
||||
if track_name:
|
||||
logger.info(
|
||||
f"Looking for artist with name {name} and track {track_name}"
|
||||
)
|
||||
keys = {}
|
||||
if name:
|
||||
name = clean_artist_name(name)
|
||||
keys["name"] = name
|
||||
|
||||
if musicbrainz_id:
|
||||
keys["musicbrainz_id"] = musicbrainz_id
|
||||
name = clean_artist_name(name)
|
||||
keys["name"] = name
|
||||
artist = cls.objects.filter(name=name).first()
|
||||
|
||||
if not keys:
|
||||
raise Exception("Must have name, mb_id or both to lookup artist")
|
||||
if artist:
|
||||
return artist
|
||||
|
||||
artist = cls.objects.filter(**keys).first()
|
||||
# alt_name = None
|
||||
artist_dict = {}
|
||||
if album_name:
|
||||
album_dict = get_album_metadata_with_artist(album_name, name)
|
||||
if album_dict:
|
||||
artist_dict = album_dict.get("primary_artist")
|
||||
if track_name:
|
||||
track_dict = get_track_metadata_with_artist(track_name, name)
|
||||
if track_dict:
|
||||
artist_dict = track_dict.get("primary_artist")
|
||||
|
||||
if not artist:
|
||||
artist = cls.objects.filter(
|
||||
models.Q(name=name) | models.Q(alt_names__icontains=name)
|
||||
).first()
|
||||
if not artist_dict:
|
||||
artist, created = cls.objects.get_or_create(name=name)
|
||||
if created:
|
||||
artist.fix_metadata()
|
||||
return artist
|
||||
|
||||
# Does not exist, look it up from Musicbrainz
|
||||
if not artist:
|
||||
alt_name = None
|
||||
try:
|
||||
artist_dict = lookup_artist_from_mb(name)
|
||||
musicbrainz_id = musicbrainz_id or artist_dict.get("id", "")
|
||||
if name != artist_dict.get("name", ""):
|
||||
alt_name = name
|
||||
name = artist_dict.get("name", "")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if musicbrainz_id:
|
||||
artist = cls.objects.filter(
|
||||
musicbrainz_id=musicbrainz_id
|
||||
).first()
|
||||
if artist and alt_name:
|
||||
if not artist.alt_names:
|
||||
artist.alt_names = alt_name
|
||||
else:
|
||||
artist.alt_names += f"\\{alt_name}"
|
||||
artist.save(update_fields=["alt_names"])
|
||||
musicbrainz_id = artist_dict.get("mbid")
|
||||
found_name = artist_dict.get("name", name)
|
||||
if found_name and name != found_name:
|
||||
alt_name = found_name
|
||||
|
||||
artist = cls.objects.filter(
|
||||
name=name, musicbrainz_id=musicbrainz_id
|
||||
).first()
|
||||
if not artist:
|
||||
artist = cls.objects.create(
|
||||
name=name, musicbrainz_id=musicbrainz_id, alt_names=alt_name
|
||||
name=found_name,
|
||||
musicbrainz_id=musicbrainz_id,
|
||||
)
|
||||
# TODO maybe this should be spun off into an async task?
|
||||
artist.fix_metadata()
|
||||
|
||||
# TODO: See if this alt_names stuff actually works or causes hard to debug problems
|
||||
# If we did find our artist, but the found name is slightly differnt, record that
|
||||
# if artist and alt_name:
|
||||
# if not artist.alt_names:
|
||||
# artist.alt_names = alt_name
|
||||
# else:
|
||||
# artist.alt_names += f"\\{alt_name}"
|
||||
# logger.info(
|
||||
# f"Add alt_name {alt_name} to artist {artist}",
|
||||
# extra={"alt_name": alt_name, "artist_id": artist.id},
|
||||
# )
|
||||
# artist.save(update_fields=["alt_names"])
|
||||
|
||||
return artist
|
||||
|
||||
|
||||
@ -319,7 +343,7 @@ class Album(TimeStampedModel):
|
||||
)
|
||||
return
|
||||
|
||||
if not self.allmusic_id or force:
|
||||
if self.album_artist and (not self.allmusic_id or force):
|
||||
slug = get_allmusic_slug(self.album_artist.name, self.name)
|
||||
if not slug:
|
||||
logger.info(
|
||||
@ -350,7 +374,12 @@ class Album(TimeStampedModel):
|
||||
logger.info(f"No data for {self} found in TheAudioDB")
|
||||
return
|
||||
|
||||
Album.objects.filter(pk=self.pk).update(**album_data)
|
||||
try:
|
||||
Album.objects.filter(pk=self.pk).update(**album_data)
|
||||
except:
|
||||
logger.info(
|
||||
f"Could not save info for album {self} with data {album_data}"
|
||||
)
|
||||
|
||||
def scrape_bandcamp(self, force=False) -> None:
|
||||
if not self.bandcamp_id or force:
|
||||
@ -489,65 +518,75 @@ class Album(TimeStampedModel):
|
||||
return f"https://bandcamp.com/search?q={album} {artist}&item_type=a"
|
||||
|
||||
@classmethod
|
||||
def find_or_create(
|
||||
cls, name: str, artist_name: str, musicbrainz_id: str = ""
|
||||
) -> "Album":
|
||||
if not name or not artist_name:
|
||||
raise Exception(
|
||||
"Must have at least name and artist name to lookup album"
|
||||
def find_or_create(cls, name: str, artist_name: str) -> "Album":
|
||||
logger.info(
|
||||
f"Looking for album with name {name} and artist_name {artist_name}"
|
||||
)
|
||||
artist = Artist.find_or_create(artist_name, album_name=name)
|
||||
album_dict = get_album_metadata_with_artist(name, artist.name)
|
||||
|
||||
if not album_dict:
|
||||
logger.info(
|
||||
f"Could not find album {name} with artist {artist.name} on musicbrainz"
|
||||
)
|
||||
|
||||
album = None
|
||||
if musicbrainz_id:
|
||||
album = cls.objects.filter(
|
||||
musicbrainz_id=musicbrainz_id,
|
||||
album, created = Album.objects.get_or_create(
|
||||
name=name,
|
||||
album_artist__name=artist_name,
|
||||
).first()
|
||||
if not album and musicbrainz_id:
|
||||
album = cls.objects.filter(
|
||||
musicbrainz_id=musicbrainz_id,
|
||||
).first()
|
||||
if not album:
|
||||
album = cls.objects.filter(
|
||||
models.Q(name=name) | models.Q(alt_names__icontains=name),
|
||||
album_artist__name=artist_name,
|
||||
).first()
|
||||
)
|
||||
if created:
|
||||
# album.fix_metadata()
|
||||
# album.fetch_artwork()
|
||||
...
|
||||
return album
|
||||
|
||||
if not album:
|
||||
alt_name = None
|
||||
try:
|
||||
album_dict = lookup_album_dict_from_mb(
|
||||
name, artist_name=artist_name
|
||||
)
|
||||
musicbrainz_id = musicbrainz_id or album_dict.get("mb_id", "")
|
||||
found_name = album_dict.get("title", "")
|
||||
if found_name and name != found_name:
|
||||
alt_name = name
|
||||
name = found_name
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
if musicbrainz_id:
|
||||
album = cls.objects.filter(
|
||||
musicbrainz_id=musicbrainz_id
|
||||
if not artist:
|
||||
artist_dict = album_dict.get("primary_artist", {})
|
||||
if artist_dict:
|
||||
artist = Artist.objects.filter(
|
||||
musicbrainz_id=artist_dict.get("mbid"),
|
||||
).first()
|
||||
if album and alt_name:
|
||||
if not album.alt_names:
|
||||
album.alt_names = alt_name
|
||||
else:
|
||||
album.alt_names += f"\\{alt_name}"
|
||||
album.save(update_fields=["alt_names"])
|
||||
if not album:
|
||||
artist = Artist.find_or_create(name=artist_name)
|
||||
album = cls.objects.create(
|
||||
name=name,
|
||||
album_artist=artist,
|
||||
musicbrainz_id=musicbrainz_id,
|
||||
alt_names=alt_name,
|
||||
)
|
||||
# TODO maybe do this in a separate process?
|
||||
album.fix_metadata()
|
||||
if not artist:
|
||||
artist = Artist.objects.create(
|
||||
musicbrainz_id=artist_dict.get("mbid"),
|
||||
)
|
||||
|
||||
extra_artists = []
|
||||
if not artist and len(album_dict.get("all_artists")) > 1:
|
||||
artist = Artist.objects.filter(name="Various Artists").first()
|
||||
extra_artists.append(artist)
|
||||
|
||||
if not artist:
|
||||
raise Exception("No album artist found, and not a compliation")
|
||||
|
||||
album = cls.objects.filter(
|
||||
models.Q(name=name) | models.Q(alt_names__icontains=name),
|
||||
album_artist=artist,
|
||||
).first()
|
||||
|
||||
alt_name = None
|
||||
found_name = album_dict.get("album_title", name)
|
||||
if found_name and name != found_name:
|
||||
alt_name = name
|
||||
|
||||
album = Album.objects.filter(
|
||||
name=found_name, musicbrainz_id=album_dict.get("mbid")
|
||||
).first()
|
||||
|
||||
if not album:
|
||||
year = None
|
||||
if album_dict.get("release_date"):
|
||||
year = album_dict.get("release_date", "").split("-")[0]
|
||||
album = Album.objects.create(
|
||||
name=found_name,
|
||||
musicbrainz_id=album_dict.get("mbid"),
|
||||
musicbrainz_releasegroup_id=album_dict.get(
|
||||
"release_group_mbid"
|
||||
),
|
||||
year=year,
|
||||
album_artist=artist,
|
||||
alt_names=alt_name,
|
||||
)
|
||||
album.artists.add(*extra_artists)
|
||||
album.fetch_artwork()
|
||||
|
||||
return album
|
||||
|
||||
@ -568,6 +607,8 @@ class Track(ScrobblableMixin):
|
||||
|
||||
@property
|
||||
def primary_album(self):
|
||||
if self.album:
|
||||
return self.album
|
||||
return self.albums.order_by("year").first()
|
||||
|
||||
def get_absolute_url(self):
|
||||
@ -594,8 +635,8 @@ class Track(ScrobblableMixin):
|
||||
url = ""
|
||||
if self.artist.thumbnail:
|
||||
url = self.artist.thumbnail_medium.url
|
||||
if self.album and self.album.cover_image:
|
||||
url = self.album.cover_image_medium.url
|
||||
if self.primary_album and self.primary_album.cover_image:
|
||||
url = self.primary_album.cover_image_medium.url
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
@ -603,9 +644,8 @@ class Track(ScrobblableMixin):
|
||||
cls,
|
||||
title: str = "",
|
||||
artist_name: str = "",
|
||||
musicbrainz_id: str = "",
|
||||
album_name: str = "",
|
||||
run_time_seconds: int = 900,
|
||||
run_time_seconds: int | None = None,
|
||||
enrich: bool = False,
|
||||
commit: bool = True,
|
||||
) -> "Track":
|
||||
@ -615,62 +655,65 @@ class Track(ScrobblableMixin):
|
||||
name
|
||||
|
||||
Optionally, we can update any found artists with overwrite."""
|
||||
created = False
|
||||
if musicbrainz_id:
|
||||
track = cls.objects.filter(musicbrainz_id=musicbrainz_id).first()
|
||||
artist = track.artist
|
||||
if not track and not (title and album_name):
|
||||
raise Exception(
|
||||
"Cannot find track with musicbrainz_id and no track title or artist name provided."
|
||||
)
|
||||
else:
|
||||
artist = Artist.find_or_create(artist_name)
|
||||
track, created = cls.objects.get_or_create(
|
||||
title=title, artist=artist
|
||||
album = None
|
||||
if album_name:
|
||||
logger.info("Looking up album for: {album_name}")
|
||||
album = Album.find_or_create(
|
||||
name=album_name, artist_name=artist_name
|
||||
)
|
||||
artist = album.album_artist
|
||||
else:
|
||||
artist = Artist.find_or_create(artist_name, track_name=title)
|
||||
if not artist:
|
||||
artist = Artist.find_or_create(artist_name)
|
||||
|
||||
if not created:
|
||||
lookup_keys = {"title": title, "artist": artist}
|
||||
if run_time_seconds:
|
||||
lookup_keys["run_time_seconds"] = run_time_seconds
|
||||
logger.info(f"Looking up track using: {lookup_keys}")
|
||||
track = cls.objects.filter(**lookup_keys).first()
|
||||
if track:
|
||||
logger.info(
|
||||
"Found exact match for track by name and artist",
|
||||
"Found match for track by name and artist, not going to musicbrainz ",
|
||||
extra={
|
||||
"track_id": track.id,
|
||||
"title": title,
|
||||
"artist_name": artist_name,
|
||||
"run_time_seconds": run_time_seconds,
|
||||
},
|
||||
)
|
||||
return track
|
||||
|
||||
track = cls.objects.filter(title=title, artist=artist).first()
|
||||
if not track:
|
||||
track, _ = cls.objects.get_or_create(title=title, artist=artist)
|
||||
|
||||
if album:
|
||||
track.albums.add(album)
|
||||
|
||||
if enrich or not track.run_time_seconds:
|
||||
logger.info(
|
||||
f"Enriching track {track}",
|
||||
extra={
|
||||
"title": title,
|
||||
"artist_name": artist_name,
|
||||
"track_id": track.id,
|
||||
},
|
||||
)
|
||||
|
||||
if track.album and album_name != track.album.name:
|
||||
# TODO found track, but it's on a different album ... associations?
|
||||
logger.info("Found track by artist, but album is different.")
|
||||
album = Album.find_or_create()
|
||||
|
||||
if enrich:
|
||||
album = None
|
||||
if album_name:
|
||||
album = Album.find_or_create(album_name)
|
||||
|
||||
if artist.musicbrainz_id:
|
||||
track_dict = lookup_track_from_mb(title, artist.musicbrainz_id)
|
||||
musicbrainz_id = musicbrainz_id or track_dict.get("id", "")
|
||||
|
||||
found_title: bool = track_dict.get("name", False)
|
||||
mismatched_title: bool = title != track_dict.get("name", "")
|
||||
if found_title and mismatched_title:
|
||||
logger.warning(
|
||||
"Source track title and found title do not match",
|
||||
extra={"title": title, "track_dict": track_dict},
|
||||
)
|
||||
|
||||
if not run_time_seconds:
|
||||
run_time_seconds = int(
|
||||
int(track_dict.get("length", 900000)) / 1000
|
||||
try:
|
||||
mbid, length = get_recording_mbid_exact(
|
||||
title, artist_name, album_name
|
||||
)
|
||||
|
||||
track.album = album
|
||||
track.artist = artist
|
||||
track.run_time_seconds = run_time_seconds
|
||||
except Exception:
|
||||
print("No musicbrainz result found, cannot enrich")
|
||||
return track
|
||||
track.run_time_seconds = run_time_seconds or int(length / 1000)
|
||||
track.musicbrainz_id = mbid
|
||||
if commit:
|
||||
track.save()
|
||||
# TODO Also set cover art and tags
|
||||
|
||||
return track
|
||||
|
||||
def fix_metadata(self, force_update=False):
|
||||
|
||||
...
|
||||
|
||||
Reference in New Issue
Block a user