From bb9936af650725275a0931e5f6fb536f95a2de3a Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Thu, 16 Feb 2023 22:31:46 -0500 Subject: [PATCH] Clean up lastfm importing --- vrobbler/apps/music/utils.py | 30 ++++++++++++------- vrobbler/apps/scrobbles/admin.py | 7 ++++- vrobbler/apps/scrobbles/lastfm.py | 25 ++++++++++++---- ...astfmimport_processed_finished_and_more.py | 23 ++++++++++++++ vrobbler/apps/scrobbles/models.py | 27 +++++++++++------ vrobbler/apps/scrobbles/views.py | 2 +- .../templates/scrobbles/scrobble_list.html | 6 ++-- 7 files changed, 89 insertions(+), 31 deletions(-) create mode 100644 vrobbler/apps/scrobbles/migrations/0019_rename_processed_on_lastfmimport_processed_finished_and_more.py diff --git a/vrobbler/apps/music/utils.py b/vrobbler/apps/music/utils.py index 109f86e..106820e 100644 --- a/vrobbler/apps/music/utils.py +++ b/vrobbler/apps/music/utils.py @@ -1,3 +1,4 @@ +import re import logging from scrobbles.musicbrainz import ( @@ -12,18 +13,27 @@ from music.models import Artist, Album, Track def get_or_create_artist(name: str, mbid: str = None) -> Artist: - if 'feat.' in name: - name = name.split('feat.')[0] - if mbid: - artist, created = Artist.objects.get_or_create(musicbrainz_id=mbid) - else: - artist, created = Artist.objects.get_or_create(name=name) + artist = None + logger.debug(f'Got artist {name} and mbid: {mbid}') - if created or not mbid: - artist_dict = lookup_artist_from_mb(name) - artist.musicbrainz_id = artist_dict["id"] + if 'feat.' in name.lower(): + name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip() + if 'featuring' in name.lower(): + name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip() + artist_dict = lookup_artist_from_mb(name) + mbid = mbid or artist_dict['id'] + + logger.debug(f'Looking up artist {name} and mbid: {mbid}') + artist, created = Artist.objects.get_or_create( + name=name, musicbrainz_id=mbid + ) + + logger.debug(f"Cleaning artist {name} with {artist_dict['name']}") + # Clean up bad names in our DB with MB names + if artist.name != artist_dict['name']: artist.name = artist_dict["name"] - artist.save(update_fields=["musicbrainz_id", "name"]) + artist.save(update_fields=["name"]) + return artist diff --git a/vrobbler/apps/scrobbles/admin.py b/vrobbler/apps/scrobbles/admin.py index 540bd8f..a5df053 100644 --- a/vrobbler/apps/scrobbles/admin.py +++ b/vrobbler/apps/scrobbles/admin.py @@ -24,7 +24,12 @@ class AudioScrobblerTSVImportAdmin(admin.ModelAdmin): @admin.register(LastFmImport) class LastFmImportAdmin(admin.ModelAdmin): date_hierarchy = "created" - list_display = ("uuid", "created", "process_count", "processed_on") + list_display = ( + "uuid", + "process_count", + "processed_finished", + "processing_started", + ) ordering = ("-created",) diff --git a/vrobbler/apps/scrobbles/lastfm.py b/vrobbler/apps/scrobbles/lastfm.py index 636c0a5..64d5043 100644 --- a/vrobbler/apps/scrobbles/lastfm.py +++ b/vrobbler/apps/scrobbles/lastfm.py @@ -124,20 +124,33 @@ class LastFM: lfm_params['limit'] = None found_scrobbles = self.user.get_recent_tracks(**lfm_params) + # TOOD spin this out into a celery task over certain threshold of found scrobbles? for scrobble in found_scrobbles: + run_time = None + run_time_ticks = None + mbid = None + artist = None + try: run_time_ticks = scrobble.track.get_duration() run_time = int(run_time_ticks / 1000) - except pylast.MalformedResponseError: - run_time_ticks = None - run_time = None - logger.warn(f"Track {scrobble.track} has no duration") + mbid = scrobble.track.get_mbid() + artist = scrobble.track.get_artist().name + except pylast.MalformedResponseError as e: + logger.warn(e) + except pylast.WSError as e: + logger.warn( + "LastFM barfed trying to get the track for {scrobble.track}" + ) + + if not mbid or not artist: + logger.warn(f"Silly LastFM, bad data, bailing on {scrobble}") + continue timestamp = datetime.utcfromtimestamp( int(scrobble.timestamp) ).replace(tzinfo=pytz.utc) - artist = scrobble.track.get_artist().name logger.info(f"{artist},{scrobble.track.title},{timestamp}") scrobbles.append( @@ -145,7 +158,7 @@ class LastFM: "artist": artist, "album": scrobble.album, "title": scrobble.track.title, - "mbid": scrobble.track.get_mbid(), + "mbid": mbid, "run_time": run_time, "run_time_ticks": run_time_ticks, "timestamp": timestamp, diff --git a/vrobbler/apps/scrobbles/migrations/0019_rename_processed_on_lastfmimport_processed_finished_and_more.py b/vrobbler/apps/scrobbles/migrations/0019_rename_processed_on_lastfmimport_processed_finished_and_more.py new file mode 100644 index 0000000..0b7c891 --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0019_rename_processed_on_lastfmimport_processed_finished_and_more.py @@ -0,0 +1,23 @@ +# Generated by Django 4.1.5 on 2023-02-16 17:13 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scrobbles', '0018_lastfmimport'), + ] + + operations = [ + migrations.RenameField( + model_name='lastfmimport', + old_name='processed_on', + new_name='processed_finished', + ), + migrations.AddField( + model_name='lastfmimport', + name='processing_started', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index b4c242a..0d8d4c9 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -82,7 +82,8 @@ class AudioScrobblerTSVImport(TimeStampedModel): class LastFmImport(TimeStampedModel): user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL) uuid = models.UUIDField(editable=False, default=uuid4) - processed_on = models.DateTimeField(**BNULL) + processing_started = models.DateTimeField(**BNULL) + processed_finished = models.DateTimeField(**BNULL) process_log = models.TextField(**BNULL) process_count = models.IntegerField(**BNULL) @@ -91,8 +92,10 @@ class LastFmImport(TimeStampedModel): def process(self, import_all=False): """Import scrobbles found on LastFM""" - if self.processed_on: - logger.info(f"{self} already processed on {self.processed_on}") + if self.processed_finished: + logger.info( + f"{self} already processed on {self.processed_finished}" + ) return last_import = None @@ -111,7 +114,10 @@ class LastFmImport(TimeStampedModel): lastfm = LastFM(self.user) last_processed = None if last_import: - last_processed = last_import.processed_on + last_processed = last_import.processed_finished + + self.processing_started = timezone.now() + self.save(update_fields=['processing_started']) scrobbles = lastfm.import_from_lastfm(last_processed) self.process_log = "" @@ -124,19 +130,22 @@ class LastFmImport(TimeStampedModel): self.process_log += log_line self.process_count = len(scrobbles) else: - self.process_log = f"Created no new scrobbles" self.process_count = 0 - self.processed_on = timezone.now() + self.processed_finished = timezone.now() self.save( - update_fields=['processed_on', 'process_count', 'process_log'] + update_fields=[ + 'processed_finished', + 'process_count', + 'process_log', + ] ) def undo(self, dryrun=False): """Undo import of scrobbles from LastFM""" LastFM.undo_lastfm_import(self.process_log, dryrun) - self.processed_on = None - self.save(update_fields=['processed_on']) + self.processed_finished = None + self.save(update_fields=['processed_finished']) class ChartRecord(TimeStampedModel): diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index 3ed3468..d40e040 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -180,7 +180,7 @@ class AudioScrobblerImportCreateView( @api_view(['GET']) def lastfm_import(request): lfm_import, created = LastFmImport.objects.get_or_create( - user=request.user, processed_on__isnull=True + user=request.user, processed_finished__isnull=True ) process_lastfm_import.delay(lfm_import.id) diff --git a/vrobbler/templates/scrobbles/scrobble_list.html b/vrobbler/templates/scrobbles/scrobble_list.html index 2b55836..6d59b5f 100644 --- a/vrobbler/templates/scrobbles/scrobble_list.html +++ b/vrobbler/templates/scrobbles/scrobble_list.html @@ -240,9 +240,7 @@ {% for scrobble in video_scrobble_list %} {{scrobble.timestamp|naturaltime}} - {% if scrobble.video.tv_series - %}S{{scrobble.video.season_number}}E{{scrobble.video.episode_number}} -{% - endif %} {{scrobble.video.title}} + {% if scrobble.video.tv_series %}S{{scrobble.video.season_number}}E{{scrobble.video.episode_number}} -{% endif %} {{scrobble.video.title}} {% if scrobble.video.tv_series %}{{scrobble.video.tv_series}}{% endif %} @@ -367,4 +365,4 @@ $('#importModal').on('shown.bs.modal', function () { $('#importInput').trigger('focus') }); $('#exportModal').on('shown.bs.modal', function () { $('#exportInput').trigger('focus') }); -{% endblock %} \ No newline at end of file +{% endblock %}