diff --git a/PROJECT.org b/PROJECT.org index 4483dab..a1da3f4 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -627,7 +627,18 @@ The Edit log form should have from top to bottom: - Expansion ids (which should a multi-select widget of expansions for this game) - Location (which should be a drop down of BoardGameLocations for this user) -** TODO [#A] Dedup track scrobbles from lastfm import :importers:lastfm:tracks: +** DONE [#A] Dedup track scrobbles from lastfm import :importers:lastfm:tracks: +:PROPERTIES: +:ID: 69f22940-8cee-4a82-b8ea-3d3d770f5e1b +:END: + +*** Description + +The historical LastFM import duplicated a number of scrobbles. The importer +itself was not fixed, but a `dedup_scrobbles` management command now removes +duplicate scrobbles where the start timestamp, end timestamp, and media +scrobbled are identical. Run it dry (default) to report duplicates, or with +`--commit` to delete them. A `--media-type` filter limits the scan. ** DONE [#A] Write a script to check for raw mopidy vs found track discrepancies :tracks:metadata: :PROPERTIES: :ID: 0b98a1b9-2848-433e-8bec-0f5737cfc74b diff --git a/tests/scrobbles_tests/test_utils.py b/tests/scrobbles_tests/test_utils.py index e1d1b07..4b245cd 100644 --- a/tests/scrobbles_tests/test_utils.py +++ b/tests/scrobbles_tests/test_utils.py @@ -1,10 +1,145 @@ from datetime import datetime + import pytz from django.contrib.auth import get_user_model +from django.utils import timezone +from music.models import Artist, Track +from scrobbles.models import Scrobble -from vrobbler.apps.scrobbles.utils import timestamp_user_tz_to_utc +from vrobbler.apps.scrobbles.utils import ( + deduplicate_scrobbles, + timestamp_user_tz_to_utc, +) + +User = get_user_model() def test_timestamp_user_tz_to_utc(): timestamp = timestamp_user_tz_to_utc(1685561082, pytz.timezone("US/Eastern")) assert timestamp == datetime(2023, 5, 31, 23, 24, 42, tzinfo=pytz.utc) + + +def _make_track(): + artist = Artist.objects.create(name="Carly Rae Jepsen") + return Track.objects.create( + title="Emotion", + artist_fk=artist, + base_run_time_seconds=60, + ) + + +def test_deduplicate_scrobbles_removes_exact_duplicates(db): + user = User.objects.create(email="dup@example.com") + track = _make_track() + timestamp = timezone.now().replace(microsecond=0) + stop_timestamp = timestamp + timezone.timedelta(seconds=60) + for _ in range(3): + Scrobble.objects.create( + user=user, + track=track, + media_type="Track", + timestamp=timestamp, + stop_timestamp=stop_timestamp, + ) + + count = deduplicate_scrobbles(commit=True) + + assert count == 2 + assert Scrobble.objects.filter(user=user, track=track).count() == 1 + + +def test_deduplicate_scrobbles_keeps_different_media(db): + user = User.objects.create(email="media@example.com") + track = _make_track() + other_track = Track.objects.create( + title="Other", + artist_fk=Artist.objects.create(name="Someone Else"), + base_run_time_seconds=60, + ) + timestamp = timezone.now().replace(microsecond=0) + stop_timestamp = timestamp + timezone.timedelta(seconds=60) + Scrobble.objects.create( + user=user, + track=track, + media_type="Track", + timestamp=timestamp, + stop_timestamp=stop_timestamp, + ) + Scrobble.objects.create( + user=user, + track=other_track, + media_type="Track", + timestamp=timestamp, + stop_timestamp=stop_timestamp, + ) + + count = deduplicate_scrobbles(commit=True) + + assert count == 0 + assert Scrobble.objects.count() == 2 + + +def test_deduplicate_scrobbles_keeps_different_timestamps(db): + user = User.objects.create(email="time@example.com") + track = _make_track() + timestamp = timezone.now().replace(microsecond=0) + stop_timestamp = timestamp + timezone.timedelta(seconds=60) + Scrobble.objects.create( + user=user, + track=track, + media_type="Track", + timestamp=timestamp, + stop_timestamp=stop_timestamp, + ) + Scrobble.objects.create( + user=user, + track=track, + media_type="Track", + timestamp=timestamp + timezone.timedelta(hours=1), + stop_timestamp=stop_timestamp + timezone.timedelta(hours=1), + ) + + count = deduplicate_scrobbles(commit=True) + + assert count == 0 + assert Scrobble.objects.count() == 2 + + +def test_deduplicate_scrobbles_dry_run_does_not_delete(db): + user = User.objects.create(email="dry@example.com") + track = _make_track() + timestamp = timezone.now().replace(microsecond=0) + stop_timestamp = timestamp + timezone.timedelta(seconds=60) + for _ in range(2): + Scrobble.objects.create( + user=user, + track=track, + media_type="Track", + timestamp=timestamp, + stop_timestamp=stop_timestamp, + ) + + count = deduplicate_scrobbles(commit=False) + + assert count == 1 + assert Scrobble.objects.filter(user=user, track=track).count() == 2 + + +def test_deduplicate_scrobbles_filters_by_media_type(db): + user = User.objects.create(email="filter@example.com") + track = _make_track() + timestamp = timezone.now().replace(microsecond=0) + stop_timestamp = timestamp + timezone.timedelta(seconds=60) + for _ in range(2): + Scrobble.objects.create( + user=user, + track=track, + media_type="Track", + timestamp=timestamp, + stop_timestamp=stop_timestamp, + ) + + count = deduplicate_scrobbles(commit=True, media_type="Video") + + assert count == 0 + assert Scrobble.objects.filter(user=user, track=track).count() == 2 diff --git a/vrobbler/apps/scrobbles/management/commands/dedup_scrobbles.py b/vrobbler/apps/scrobbles/management/commands/dedup_scrobbles.py new file mode 100644 index 0000000..2e18df4 --- /dev/null +++ b/vrobbler/apps/scrobbles/management/commands/dedup_scrobbles.py @@ -0,0 +1,32 @@ +from django.core.management.base import BaseCommand + +from vrobbler.apps.scrobbles.utils import deduplicate_scrobbles + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--commit", + action="store_true", + help="Commit the deletes", + ) + parser.add_argument( + "--media-type", + default=None, + help="Only dedupe scrobbles of this media type (e.g. Track)", + ) + + def handle(self, *args, **options): + commit = options["commit"] + media_type = options["media_type"] + if not commit: + self.stdout.write("No changes will be saved, use --commit to save") + + count = deduplicate_scrobbles(commit=commit, media_type=media_type) + + if commit: + self.stdout.write( + self.style.SUCCESS(f"Deleted {count} duplicate scrobbles") + ) + else: + self.stdout.write(f"Found {count} duplicate scrobbles") diff --git a/vrobbler/apps/scrobbles/utils.py b/vrobbler/apps/scrobbles/utils.py index 13a5ae3..50621be 100644 --- a/vrobbler/apps/scrobbles/utils.py +++ b/vrobbler/apps/scrobbles/utils.py @@ -90,7 +90,8 @@ def get_scrobbles_for_media(media_obj, user: User) -> models.QuerySet: return Scrobble.objects.filter(media_query, user=user) -def get_recently_played_board_games(user: User) -> dict: ... +def get_recently_played_board_games(user: User) -> dict: + ... def get_long_play_media_model(app_label: str, model_name: str): @@ -240,6 +241,46 @@ def delete_zombie_scrobbles(dry_run=True): return zombies_found +def deduplicate_scrobbles(commit=True, media_type=None): + """Delete scrobbles duplicated by the Last.fm historical import. + + A duplicate is any scrobble whose (user, timestamp, stop_timestamp, media) + combination matches another scrobble. The oldest scrobble in each duplicate + group is kept and the rest are deleted. Passing ``media_type`` limits the + scan to a single media type. Returns the number of duplicate scrobbles + found. + """ + from scrobbles.models import TYPE_FK_PREFETCHES, Scrobble + + media_types = [media_type] if media_type else list(TYPE_FK_PREFETCHES) + group_cols = ("user_id", "timestamp", "stop_timestamp") + duplicate_ids = [] + + for mtype in media_types: + fk_field = TYPE_FK_PREFETCHES[mtype][0] + scrobbles = Scrobble.objects.filter( + media_type=mtype, **{f"{fk_field}__isnull": False} + ) + groups = ( + scrobbles.values(*group_cols, f"{fk_field}_id") + .annotate(count=models.Count("id")) + .filter(count__gt=1) + ) + for group in groups: + group_filter = {col: group[col] for col in (*group_cols, f"{fk_field}_id")} + keep_id = scrobbles.filter(**group_filter).order_by("id").first().id + duplicate_ids.extend( + scrobbles.filter(**group_filter) + .exclude(id=keep_id) + .values_list("id", flat=True) + ) + + if duplicate_ids and commit: + Scrobble.objects.filter(id__in=duplicate_ids).delete() + + return len(duplicate_ids) + + def import_from_webdav_for_all_users(restart=False): """Grab a list of all users with WebDAV enabled and kickoff imports for them""" from books.koreader import fetch_file_from_webdav @@ -356,7 +397,10 @@ def apply_media_unit_conversions(form, media_type: str, user) -> str: drink_types = ("Beer", "Wine", "Coffee") if media_type in drink_types and "size_ml" in form.fields: - use_oz = getattr(getattr(user, "profile", None), "volume_unit", "metric") == "imperial" + use_oz = ( + getattr(getattr(user, "profile", None), "volume_unit", "metric") + == "imperial" + ) if use_oz: form.fields["size_ml"].label = "Size (oz)" original_clean = form.fields["size_ml"].clean