From e8e989bb6306865704f4af15ab45fdf3776645ee Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Sun, 20 Jul 2025 21:18:46 -0400 Subject: [PATCH] [music] Add albums to tracks and utility to condense tracks --- .../music/migrations/0027_track_albums.py | 20 +++++ vrobbler/apps/music/models.py | 62 +++++++++----- vrobbler/apps/music/utils.py | 85 +++++++++++++++++++ .../management/commands/condense_albums.py | 19 +++++ .../management/commands/dedup_tracks.py | 4 +- vrobbler/apps/scrobbles/utils.py | 36 +------- 6 files changed, 170 insertions(+), 56 deletions(-) create mode 100644 vrobbler/apps/music/migrations/0027_track_albums.py create mode 100644 vrobbler/apps/scrobbles/management/commands/condense_albums.py diff --git a/vrobbler/apps/music/migrations/0027_track_albums.py b/vrobbler/apps/music/migrations/0027_track_albums.py new file mode 100644 index 0000000..eb41055 --- /dev/null +++ b/vrobbler/apps/music/migrations/0027_track_albums.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.19 on 2025-07-20 20:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("music", "0026_album_alt_names"), + ] + + operations = [ + migrations.AddField( + model_name="track", + name="albums", + field=models.ManyToManyField( + blank=True, null=True, related_name="tracks", to="music.album" + ), + ), + ] diff --git a/vrobbler/apps/music/models.py b/vrobbler/apps/music/models.py index 4c8ae64..225466b 100644 --- a/vrobbler/apps/music/models.py +++ b/vrobbler/apps/music/models.py @@ -14,7 +14,12 @@ from imagekit.models import ImageSpecField from imagekit.processors import ResizeToFit from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic from music.bandcamp import get_bandcamp_slug -from music.musicbrainz import lookup_album_dict_from_mb, lookup_album_from_mb, lookup_track_from_mb, lookup_artist_from_mb +from music.musicbrainz import ( + lookup_album_dict_from_mb, + lookup_album_from_mb, + lookup_track_from_mb, + lookup_artist_from_mb, +) from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb from music.utils import clean_artist_name from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin @@ -22,6 +27,7 @@ from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin logger = logging.getLogger(__name__) BNULL = {"blank": True, "null": True} + class Artist(TimeStampedModel): """Represents a music artist. @@ -170,7 +176,9 @@ class Artist(TimeStampedModel): return f"https://bandcamp.com/search?q={artist}&item_type=b" @classmethod - def find_or_create(cls, name: str = "", musicbrainz_id: str = "") -> "Artist": + def find_or_create( + cls, name: str = "", musicbrainz_id: str = "" + ) -> "Artist": keys = {} if name: name = clean_artist_name(name) @@ -547,12 +555,8 @@ class Album(TimeStampedModel): class Track(ScrobblableMixin): COMPLETION_PERCENT = getattr(settings, "MUSIC_COMPLETION_PERCENT", 100) - class Opinion(models.IntegerChoices): - DOWN = -1, "Thumbs down" - NEUTRAL = 0, "No opinion" - UP = 1, "Thumbs up" - artist = models.ForeignKey(Artist, on_delete=models.DO_NOTHING) + albums = models.ManyToManyField(Album, related_name="tracks") album = models.ForeignKey(Album, on_delete=models.DO_NOTHING, **BNULL) musicbrainz_id = models.CharField(max_length=255, **BNULL) @@ -562,6 +566,10 @@ class Track(ScrobblableMixin): def __str__(self): return f"{self.title} by {self.artist}" + @property + def primary_album(self): + return self.albums.order_by("year").first() + def get_absolute_url(self): return reverse("music:track_detail", kwargs={"slug": self.uuid}) @@ -590,17 +598,16 @@ class Track(ScrobblableMixin): url = self.album.cover_image_medium.url return url - @classmethod def find_or_create( - cls, - title: str = "", - artist_name: str = "", - musicbrainz_id: str = "", - album_name: str = "", - run_time_seconds: int = 900, - enrich: bool = False, - commit: bool = True + cls, + title: str = "", + artist_name: str = "", + musicbrainz_id: str = "", + album_name: str = "", + run_time_seconds: int = 900, + enrich: bool = False, + commit: bool = True, ) -> "Track": """Given a name, try to find the track by the artist from Musicbrainz. @@ -613,13 +620,24 @@ class Track(ScrobblableMixin): track = cls.objects.filter(musicbrainz_id=musicbrainz_id).first() artist = track.artist if not track and not (title and album_name): - raise Exception("Cannot find track with musicbrainz_id and no track title or artist name provided.") + raise Exception( + "Cannot find track with musicbrainz_id and no track title or artist name provided." + ) else: artist = Artist.find_or_create(artist_name) - track, created = cls.objects.get_or_create(title=title, artist=artist) + track, created = cls.objects.get_or_create( + title=title, artist=artist + ) if not created: - logger.info("Found exact match for track by name and artist", extra={"title": title, "artist_name": artist_name, "track_id": track.id}) + logger.info( + "Found exact match for track by name and artist", + extra={ + "title": title, + "artist_name": artist_name, + "track_id": track.id, + }, + ) if track.album and album_name != track.album.name: # TODO found track, but it's on a different album ... associations? @@ -638,14 +656,16 @@ class Track(ScrobblableMixin): found_title: bool = track_dict.get("name", False) mismatched_title: bool = title != track_dict.get("name", "") if found_title and mismatched_title: - logger.warning("Source track title and found title do not match", extra={"title": title, "track_dict": track_dict}) + logger.warning( + "Source track title and found title do not match", + extra={"title": title, "track_dict": track_dict}, + ) if not run_time_seconds: run_time_seconds = int( int(track_dict.get("length", 900000)) / 1000 ) - track.album = album track.artist = artist track.run_time_seconds = run_time_seconds diff --git a/vrobbler/apps/music/utils.py b/vrobbler/apps/music/utils.py index 707fcea..27d7f65 100644 --- a/vrobbler/apps/music/utils.py +++ b/vrobbler/apps/music/utils.py @@ -1,10 +1,12 @@ import logging import re +from django.db import IntegrityError, models, transaction from music.constants import VARIOUS_ARTIST_DICT logger = logging.getLogger(__name__) + def clean_artist_name(name: str) -> str: """Remove featured names from artist string.""" if "feat." in name.lower(): @@ -16,6 +18,7 @@ def clean_artist_name(name: str) -> str: return name + def get_or_create_various_artists() -> "Artist": from music.models import Artist @@ -24,3 +27,85 @@ def get_or_create_various_artists() -> "Artist": artist = Artist.objects.create(**VARIOUS_ARTIST_DICT) logger.info("Created Various Artists placeholder") return artist + + +def deduplicate_tracks(commit=False) -> int: + from music.models import Track + + duplicates = ( + Track.objects.values("artist", "title") + .annotate(dup_count=models.Count("id")) + .filter(dup_count__gt=1) + ) + + query = models.Q() + for dup in duplicates: + query |= models.Q(artist=dup["artist"], title=dup["title"]) + + duplicate_tracks = Track.objects.filter(query) + + for b in duplicate_tracks: + tracks = Track.objects.filter(artist=b.artist, title=b.title) + first = tracks.first() + for other in tracks.exclude(id=first.id): + print("Moving scrobbles for", other.id, " to ", first.id) + if commit: + with transaction.atomic(): + other.scrobble_set.update(track=first) + print("deleting ", other.id, " - ", other) + try: + other.delete() + except IntegrityError as e: + print( + "could not delete ", + other.id, + f": IntegrityError {e}", + ) + return len(duplicate_tracks) + + +def condense_albums(commit: bool = False): + from music.models import Track + from scrobbles.models import Scrobble + + processed_ids = [] + for track in Track.objects.all(): + albums_to_add = [] + duplicates = ( + Track.objects.filter(title=track.title, artist=track.artist) + .exclude(id=track.id) + .exclude(id__in=processed_ids) + ) + + if commit and track.album: + albums_to_add.append(track.album) + + for dup_track in duplicates: + logger.info(f"Adding {dup_track.album} to {track} albums") + if commit and dup_track.album: + track.albums.add(dup_track.album) + + # Find out if this track appears more than once + duplicates = Track.objects.filter( + title=track.title, artist=track.artist + ) + if duplicates.count() > 1: + logger.info(f"Track appears more than once, condensing: {track}") + + albums_to_add.extend([d.album for d in duplicates]) + # Find all scrobbles + duplicate_ids = duplicates.values_list("id", flat=True) + scrobbles = Scrobble.objects.filter(track_id__in=duplicate_ids) + logger.info( + f"Found {scrobbles.count()} scrobbles to merge onto {track}" + ) + if commit: + scrobbles.update(track=track) + track.albums.add(*list(set(albums_to_add))) + + processed_ids.extend(duplicate_ids) + + if commit: + Track.objects.filter(scrobble__isnull=True).delete() + + return len(set(processed_ids)) diff --git a/vrobbler/apps/scrobbles/management/commands/condense_albums.py b/vrobbler/apps/scrobbles/management/commands/condense_albums.py new file mode 100644 index 0000000..c04eafd --- /dev/null +++ b/vrobbler/apps/scrobbles/management/commands/condense_albums.py @@ -0,0 +1,19 @@ +from django.core.management.base import BaseCommand +from vrobbler.apps.music.utils import condense_albums + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--commit", + action="store_true", + help="Commit changes", + ) + + def handle(self, *args, **options): + commit = False + if options["commit"]: + commit = True + print(f"Condensing albums") + album_count = condense_albums(commit) + print(f"Condensed {album_count} albums") diff --git a/vrobbler/apps/scrobbles/management/commands/dedup_tracks.py b/vrobbler/apps/scrobbles/management/commands/dedup_tracks.py index b552b28..0c798f6 100644 --- a/vrobbler/apps/scrobbles/management/commands/dedup_tracks.py +++ b/vrobbler/apps/scrobbles/management/commands/dedup_tracks.py @@ -1,5 +1,5 @@ from django.core.management.base import BaseCommand -from vrobbler.apps.scrobbles.utils import deduplicate_tracks +from vrobbler.apps.music.utils import deduplicate_tracks class Command(BaseCommand): @@ -14,5 +14,7 @@ class Command(BaseCommand): commit = False if options["commit"]: commit = True + else: + print("No changes will be saved, use --commit to save") dups = deduplicate_tracks(commit=commit) print(f"Deduplicated {dups} music tracks") diff --git a/vrobbler/apps/scrobbles/utils.py b/vrobbler/apps/scrobbles/utils.py index 41bdef4..ab205ce 100644 --- a/vrobbler/apps/scrobbles/utils.py +++ b/vrobbler/apps/scrobbles/utils.py @@ -2,17 +2,17 @@ import hashlib import logging import re from datetime import datetime, timedelta, tzinfo -from sqlite3 import IntegrityError from urllib.parse import urlparse import pytz from django.apps import apps from django.contrib.auth import get_user_model -from django.db import models, transaction +from django.db import models from django.utils import timezone from profiles.models import UserProfile from profiles.utils import now_user_timezone from scrobbles.constants import LONG_PLAY_MEDIA +from scrobbles.notifications import NtfyNotification from scrobbles.tasks import ( process_koreader_import, process_lastfm_import, @@ -20,8 +20,6 @@ from scrobbles.tasks import ( ) from webdav.client import get_webdav_client -from vrobbler.apps.scrobbles.notifications import NtfyNotification - logger = logging.getLogger(__name__) User = get_user_model() @@ -275,36 +273,6 @@ def get_file_md5_hash(file_path: str) -> str: return file_hash.hexdigest() -def deduplicate_tracks(commit=False) -> int: - from music.models import Track - - # TODO This whole thing should iterate over users - dups = [] - - for t in Track.objects.all(): - if Track.objects.filter(title=t.title, artist=t.artist).exists(): - dups.append(t) - - for b in dups: - tracks = Track.objects.filter(artist=b.artist, title=b.title) - first = tracks.first() - for other in tracks.exclude(id=first.id): - print("moving scrobbles for ", other.id, " to ", first.id) - if commit: - with transaction.atomic(): - other.scrobble_set.update(track=first) - print("deleting ", other.id, " - ", other) - try: - other.delete() - except IntegrityError as e: - print( - "could not delete ", - other.id, - f": IntegrityError {e}", - ) - return len(dups) - - def send_stop_notifications_for_in_progress_scrobbles() -> int: """Get all inprogress scrobbles and check if they're passed their media obj length.