From 278cab32ea7f772b5aff583c79b670752bf73367 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Tue, 5 Aug 2025 02:01:31 -0400 Subject: [PATCH] [scrobbles] Start cleaning up logdata --- PROJECT.org | 4 + vrobbler/apps/music/models.py | 20 ++-- vrobbler/apps/people/models.py | 3 + .../commands/convert_task_log_data.py | 25 +++++ vrobbler/apps/scrobbles/mixins.py | 13 +++ vrobbler/apps/scrobbles/models.py | 7 +- vrobbler/apps/scrobbles/scrobblers.py | 50 ++++++---- vrobbler/apps/tasks/models.py | 29 ++---- vrobbler/apps/tasks/utils.py | 94 ++++++++++++++++++- 9 files changed, 191 insertions(+), 54 deletions(-) create mode 100644 vrobbler/apps/scrobbles/management/commands/convert_task_log_data.py diff --git a/PROJECT.org b/PROJECT.org index 266b159..25a038a 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -443,6 +443,10 @@ it's annoying. ** TODO [#C] Allow users to see tasks on calendar view :vrobbler:personal:project:templates:feature: https://codepen.io/oliviale/pen/QYqybo ** TODO [#C] Come up with a possible flow using WebDAV and super-productivity for tasks :personal:feature:project:vrobbler:tasks: +** DONE Fix long play scrobbles to provide better data :vrobbler:feature:scrobbles:longplay:personal:project: +:PROPERTIES: +:ID: 99f6bd77-dc8f-6ed1-0321-32a52c944264 +:END: * Version 19.0 ** DONE Add periodic check for mood :vrobbler:feature:moods:personal:project: :PROPERTIES: diff --git a/vrobbler/apps/music/models.py b/vrobbler/apps/music/models.py index 731138d..f46f9f8 100644 --- a/vrobbler/apps/music/models.py +++ b/vrobbler/apps/music/models.py @@ -1,5 +1,6 @@ import logging -from typing import Dict, Optional +from dataclasses import dataclass +from typing import Optional from uuid import uuid4 import musicbrainzngs @@ -15,24 +16,26 @@ from imagekit.processors import ResizeToFit from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic from music.bandcamp import get_bandcamp_slug from music.musicbrainz import ( - get_album_metadata, get_album_metadata_with_artist, - get_artist_metadata_extended, get_recording_mbid_exact, get_track_metadata_with_artist, - lookup_album_dict_from_mb, - lookup_album_from_mb, - lookup_track_from_mb, - lookup_artist_from_mb, ) from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb from music.utils import clean_artist_name +from scrobbles.dataclasses import BaseLogData from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin logger = logging.getLogger(__name__) BNULL = {"blank": True, "null": True} +@dataclass +class TrackLogData(BaseLogData): + mopidy_source: Optional[str] = None + rockbox_info: Optional[str] = None + rating: Optional[int] = None + + class Artist(TimeStampedModel): """Represents a music artist. @@ -605,6 +608,9 @@ class Track(ScrobblableMixin): def __str__(self): return f"{self.title} by {self.artist}" + def logdata_cls(self): + return TrackLogData + @property def primary_album(self): if self.album: diff --git a/vrobbler/apps/people/models.py b/vrobbler/apps/people/models.py index 4c739a7..3f01094 100644 --- a/vrobbler/apps/people/models.py +++ b/vrobbler/apps/people/models.py @@ -15,3 +15,6 @@ class Person(TimeStampedModel): bgg_username = models.CharField(max_length=100, **BNULL) lichess_username = models.CharField(max_length=100, **BNULL) bio = models.TextField(**BNULL) + + def __str__(self): + return self.name diff --git a/vrobbler/apps/scrobbles/management/commands/convert_task_log_data.py b/vrobbler/apps/scrobbles/management/commands/convert_task_log_data.py new file mode 100644 index 0000000..2776941 --- /dev/null +++ b/vrobbler/apps/scrobbles/management/commands/convert_task_log_data.py @@ -0,0 +1,25 @@ +from django.core.management.base import BaseCommand +from vrobbler.apps.tasks.utils import ( + convert_notes_to_dict, + convert_old_boardgame_log_to_new, + convert_old_orgmode_log_to_new, +) + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument( + "--commit", + action="store_true", + help="Commit changes", + ) + + def handle(self, *args, **options): + commit = False + if options["commit"]: + commit = True + else: + print("No changes will be saved, use --commit to save") + convert_old_orgmode_log_to_new(commit) + convert_notes_to_dict(commit) + convert_old_boardgame_log_to_new(commit) diff --git a/vrobbler/apps/scrobbles/mixins.py b/vrobbler/apps/scrobbles/mixins.py index 0bc6445..52c4565 100644 --- a/vrobbler/apps/scrobbles/mixins.py +++ b/vrobbler/apps/scrobbles/mixins.py @@ -65,6 +65,10 @@ class ScrobblableMixin(TimeStampedModel): class Meta: abstract = True + @classmethod + def is_long_play_media(cls) -> bool: + return False + def scrobble_for_user( self, user_id, @@ -136,6 +140,15 @@ class LongPlayScrobblableMixin(ScrobblableMixin): class Meta: abstract = True + @classmethod + def is_long_play_media(cls) -> bool: + return True + + def is_complete(self) -> bool: + if self.log: + return bool(self.log.get("long_play_complete", None)) + return False + def get_longplay_finish_url(self): return reverse("scrobbles:longplay-finish", kwargs={"uuid": self.uuid}) diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 8e3c503..1e82254 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -738,7 +738,7 @@ class Scrobble(TimeStampedModel): log_dict = {} try: - return logdata_cls.from_dict(log_dict) + return logdata_cls(**log_dict) except ParseError as e: logger.warning( "Could not parse log data", @@ -749,6 +749,8 @@ class Scrobble(TimeStampedModel): }, ) return logdata_cls() + except TypeError as e: + return logdata_cls() def redirect_url(self, user_id) -> str: user = User.objects.filter(id=user_id).first() @@ -1065,8 +1067,7 @@ class Scrobble(TimeStampedModel): return media_obj def __str__(self): - timestamp = self.timestamp.strftime("%Y-%m-%d") - return f"Scrobble of {self.media_obj} ({timestamp})" + return f"Scrobble of {self.media_obj} ({self.timestamp})" def calc_reading_duration(self) -> int: duration = 0 diff --git a/vrobbler/apps/scrobbles/scrobblers.py b/vrobbler/apps/scrobbles/scrobblers.py index d99d0fb..93211b0 100644 --- a/vrobbler/apps/scrobbles/scrobblers.py +++ b/vrobbler/apps/scrobbles/scrobblers.py @@ -411,7 +411,7 @@ def email_scrobble_board_game( except IndexError: second = 0 - log_data["details"] = play_dict.get("comments") + log_data["notes"] = [play_dict.get("comments")] log_data["expansion_ids"] = [] try: base_game = base_games[play_dict.get("gameRefId")] @@ -587,9 +587,9 @@ def todoist_scrobble_update_task( ) return - existing_notes = scrobble.log.get("notes", {}) - existing_notes[todoist_note.get("todoist_id")] = todoist_note.get("notes") - scrobble.log["notes"] = existing_notes + if not scrobble.log.get("notes"): + scrobble.log["notes"] = [] + scrobble.log["notes"].append(todoist_note.get("notes")) scrobble.save(update_fields=["log"]) logger.info( "[todoist_scrobble_update_task] todoist note added", @@ -615,7 +615,7 @@ def todoist_scrobble_task( ) task = Task.find_or_create(title) - timestamp = pendulum.parse(todoist_task.get("updated_at", timezone.now())) + timestamp = pendulum.parse(todoist_task.pop("updated_at", timezone.now())) in_progress_scrobble = Scrobble.objects.filter( user_id=user_id, in_progress=True, @@ -657,8 +657,12 @@ def todoist_scrobble_task( ) return todoist_scrobble_task_finish(todoist_task, user_id, timestamp) - # Default to create new scrobble "if not in_progress_scrobble and in_progress_in_todoist" - # TODO Should use updated_at from TOdoist, but parsing isn't working + todoist_task["title"] = todoist_task.pop("description") + todoist_task["description"] = todoist_task.pop("details") + todoist_task["labels"] = todoist_task.pop("todoist_label_list", []) + todoist_task.pop("todoist_type") + todoist_task.pop("todoist_event") + scrobble_dict = { "user_id": user_id, "timestamp": timestamp, @@ -686,8 +690,8 @@ def emacs_scrobble_update_task( scrobble = Scrobble.objects.filter( in_progress=True, user_id=user_id, - log__source_id=emacs_id, - log__source="orgmode", + log__orgmode_id=emacs_id, + source="Org-mode", ).first() if not scrobble: @@ -736,18 +740,18 @@ def emacs_scrobble_task( stopped: bool = False, user_context_list: list[str] = [], ) -> Scrobble | None: - source_id = task_data.get("source_id") + orgmode_id = task_data.get("source_id") title = get_title_from_labels( task_data.get("labels", []), user_context_list ) task = Task.find_or_create(title) - timestamp = pendulum.parse(task_data.get("updated_at", timezone.now())) + timestamp = pendulum.parse(task_data.pop("updated_at", timezone.now())) in_progress_scrobble = Scrobble.objects.filter( user_id=user_id, in_progress=True, - log__source_id=source_id, + log__orgmode_id=orgmode_id, log__source="orgmode", task=task, ).last() @@ -756,7 +760,7 @@ def emacs_scrobble_task( logger.info( "[emacs_scrobble_task] cannot stop already stopped task", extra={ - "emacs_id": source_id, + "orgmode_id": orgmode_id, }, ) return @@ -765,7 +769,7 @@ def emacs_scrobble_task( logger.info( "[emacs_scrobble_task] cannot start already started task", extra={ - "emacs_id": source_id, + "ormode_id": orgmode_id, }, ) return in_progress_scrobble @@ -775,7 +779,7 @@ def emacs_scrobble_task( logger.info( "[emacs_scrobble_task] finishing", extra={ - "emacs_id": source_id, + "orgmode_id": orgmode_id, }, ) in_progress_scrobble.stop(timestamp=timestamp, force_finish=True) @@ -786,11 +790,17 @@ def emacs_scrobble_task( notes = task_data.pop("notes") if notes: - task_data["notes"] = [] - for note in notes: - task_data["notes"].append( - {note.get("timestamp"): note.get("content")} - ) + task_data["notes"] = [note.get("content") for note in notes] + task_data["title"] = task_data.pop("description") + task_data["description"] = task_data.pop("body") + task_data["labels"] = task_data.pop("labels") + + task_data["orgmode_id"] = task_data.pop("source_id") + task_data["orgmode_state"] = task_data.pop("state") + task_data["orgmode_properties"] = task_data.pop("properties") + task_data["orgmode_drawers"] = task_data.pop("drawers") + task_data["orgmode_timestamps"] = task_data.pop("timestamps") + task_data.pop("source") scrobble_dict = { "user_id": user_id, diff --git a/vrobbler/apps/tasks/models.py b/vrobbler/apps/tasks/models.py index 1cfabc3..155e603 100644 --- a/vrobbler/apps/tasks/models.py +++ b/vrobbler/apps/tasks/models.py @@ -5,7 +5,7 @@ from typing import Optional from django.apps import apps from django.db import models from django.urls import reverse -from scrobbles.dataclasses import JSONDataclass +from scrobbles.dataclasses import BaseLogData, JSONDataclass from scrobbles.mixins import LongPlayScrobblableMixin, ScrobblableConstants BNULL = {"blank": True, "null": True} @@ -14,27 +14,18 @@ TODOIST_TASK_URL = "https://app.todoist.com/app/task/{id}" @dataclass -class TaskLogData(JSONDataclass): - description: Optional[str] = None +class TaskLogData(BaseLogData): title: Optional[str] = None - project: Optional[str] = None - notes: Optional[dict] = None - updated_at: Optional[str] = None + labels: Optional[list[str]] = None + + orgmode_id: Optional[str] = None + orgmode_state: Optional[str] = None + orgmode_properties: Optional[dict] = None + orgmode_drawers: Optional[list] = None + orgmode_timestamps: Optional[list] = None + todoist_id: Optional[str] = None - todoist_event: Optional[str] = None - todoist_type: Optional[str] = None - todoist_type: Optional[str] = None - todoist_label_list: Optional[list] = None todoist_project_id: Optional[str] = None - body: Optional[str] = None - state: Optional[str] = None - labels: Optional[str] = None - properties: Optional[list] = None - drawers: Optional[list] = None - source: Optional[str] = None - source_id: Optional[str] = None - timestamps: Optional[list] = None - details: Optional[str] = None def notes_as_str(self) -> str: """Return formatted notes with line breaks and no keys""" diff --git a/vrobbler/apps/tasks/utils.py b/vrobbler/apps/tasks/utils.py index a3aff9d..43a1be0 100644 --- a/vrobbler/apps/tasks/utils.py +++ b/vrobbler/apps/tasks/utils.py @@ -1,22 +1,106 @@ import logging +from datetime import timedelta from django.conf import settings +from scrobbles.models import Scrobble logger = logging.getLogger(__name__) -def get_title_from_labels(labels: list[str], user_context_labels: list[str] = []) -> str: + +def get_title_from_labels( + labels: list[str], user_context_labels: list[str] = [] +) -> str: title = "Unknown" - task_context_labels: list = user_context_labels or settings.DEFAULT_TASK_CONTEXT_TAG_LIST for label in labels: - # TODO We may also want to take a user list of labels instead label = label.capitalize() - if label in task_context_labels: + if label in user_context_labels: title = label continue if title == "Unknown": logger.warning( "Missing a configured title context for task", - extra={"labels": labels, "task_context_labels": task_context_labels}, + extra={ + "labels": labels, + "user_context_labels": user_context_labels, + }, ) return title + + +def convert_old_orgmode_log_to_new(commit=False): + scrobbles = Scrobble.objects.filter( + source="Org-mode", log__has_key="drawers" + ) + for scrobble in scrobbles: + scrobble.log["title"] = scrobble.log.pop("description") + scrobble.log["description"] = scrobble.log.pop("details") + + scrobble.log["orgmode_body"] = scrobble.log.pop("body") + scrobble.log["orgmode_state"] = scrobble.log.pop("state") + scrobble.log["orgmode_properties"] = scrobble.log.pop("properties") + scrobble.log["orgmode_drawers"] = scrobble.log.pop("drawers") + scrobble.log["orgmode_timestamps"] = scrobble.log.pop("timestamps") + scrobble.log["orgmode_id"] = scrobble.log.pop("source_id") + + if commit: + scrobble.save(update_fields=["log"]) + print(f"Updated {scrobbles.count()} orgmode tasks logs") + + +def convert_old_todoist_log_to_new(commit=False): + scrobbles = Scrobble.objects.filter( + source="Todoist", log__has_key="todoist_type" + ) + for scrobble in scrobbles: + scrobble.log["title"] = scrobble.log.pop("description") + scrobble.log["description"] = scrobble.log.pop("details") + scrobble.log["todoist_id"] = scrobble.log.pop("source_id") + scrobble.log["labels"] = scrobble.log.pop("todoist_label_list") + + scrobble.log.pop("todoist_type") + scrobble.log.pop("todoist_event") + + print(f"Updating scrobble {scrobble.id}") + if commit: + scrobble.save(update_fields=["log"]) + print(f"Converted {scrobbles.count()} todoist tasks logs") + + +def convert_notes_to_dict(commit=False): + scrobbles = Scrobble.objects.filter(log__notes__isnull=False) + count = 0 + for scrobble in scrobbles: + if isinstance(scrobble.log, str): + print(f"Converting {scrobble} string note to dict") + if scrobble.log.get("notes") == "": + scrobble.log.pop("notes") + key = str(int(scrobble.timestamp.timestamp())) + notes = scrobble.log.pop("notes") + scrobble.log = {} + scrobble.log["notes"] = {key: notes} + count += 1 + + if isinstance(scrobble.log.get("notes"), list): + note_list = scrobble.log.pop("notes") + if all(isinstance(item, dict) for item in note_list): + scrobble.log["notes"] = [ + value for d in note_list for value in d.values() + ] + count += 1 + if commit: + scrobble.save(update_fields=["log"]) + print(f"Updated {count} todoist tasks scrobbles") + + +def convert_old_boardgame_log_to_new(commit=False): + scrobbles = Scrobble.objects.filter( + board_game__isnull=False, log__has_key="notes" + ) + for scrobble in scrobbles: + if isinstance(scrobble.log.get("notes"), str): + scrobble.log["notes"] = [scrobble.log.pop("notes")] + + if commit: + scrobble.save(update_fields=["log"]) + print(f"Updated {scrobbles.count()} board game scrobbles")