[scrobbles] Start cleaning up logdata

PROJECT.org

@@ -443,6 +443,10 @@ it's annoying.
 ** TODO [#C] Allow users to see tasks on calendar view :vrobbler:personal:project:templates:feature:
 https://codepen.io/oliviale/pen/QYqybo
 ** TODO [#C] Come up with a possible flow using WebDAV and super-productivity for tasks :personal:feature:project:vrobbler:tasks:
+** DONE Fix long play scrobbles to provide better data :vrobbler:feature:scrobbles:longplay:personal:project:
+:PROPERTIES:
+:ID:       99f6bd77-dc8f-6ed1-0321-32a52c944264
+:END:
 * Version 19.0
 ** DONE Add periodic check for mood :vrobbler:feature:moods:personal:project:
 :PROPERTIES:

vrobbler/apps/music/models.py

@@ -1,5 +1,6 @@
 import logging
-from typing import Dict, Optional
+from dataclasses import dataclass
+from typing import Optional
 from uuid import uuid4
 
 import musicbrainzngs
@@ -15,24 +16,26 @@ from imagekit.processors import ResizeToFit
 from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic
 from music.bandcamp import get_bandcamp_slug
 from music.musicbrainz import (
-    get_album_metadata,
     get_album_metadata_with_artist,
-    get_artist_metadata_extended,
     get_recording_mbid_exact,
     get_track_metadata_with_artist,
-    lookup_album_dict_from_mb,
-    lookup_album_from_mb,
-    lookup_track_from_mb,
-    lookup_artist_from_mb,
 )
 from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb
 from music.utils import clean_artist_name
+from scrobbles.dataclasses import BaseLogData
 from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
 
 logger = logging.getLogger(__name__)
 BNULL = {"blank": True, "null": True}
 
 
+@dataclass
+class TrackLogData(BaseLogData):
+    mopidy_source: Optional[str] = None
+    rockbox_info: Optional[str] = None
+    rating: Optional[int] = None
+
+
 class Artist(TimeStampedModel):
     """Represents a music artist.
 
@@ -605,6 +608,9 @@ class Track(ScrobblableMixin):
     def __str__(self):
         return f"{self.title} by {self.artist}"
 
+    def logdata_cls(self):
+        return TrackLogData
+
     @property
     def primary_album(self):
         if self.album:

vrobbler/apps/people/models.py

@@ -15,3 +15,6 @@ class Person(TimeStampedModel):
     bgg_username = models.CharField(max_length=100, **BNULL)
     lichess_username = models.CharField(max_length=100, **BNULL)
     bio = models.TextField(**BNULL)
+
+    def __str__(self):
+        return self.name

vrobbler/apps/scrobbles/management/commands/convert_task_log_data.py

@@ -0,0 +1,25 @@
+from django.core.management.base import BaseCommand
+from vrobbler.apps.tasks.utils import (
+    convert_notes_to_dict,
+    convert_old_boardgame_log_to_new,
+    convert_old_orgmode_log_to_new,
+)
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--commit",
+            action="store_true",
+            help="Commit changes",
+        )
+
+    def handle(self, *args, **options):
+        commit = False
+        if options["commit"]:
+            commit = True
+        else:
+            print("No changes will be saved, use --commit to save")
+        convert_old_orgmode_log_to_new(commit)
+        convert_notes_to_dict(commit)
+        convert_old_boardgame_log_to_new(commit)

vrobbler/apps/scrobbles/mixins.py

@@ -65,6 +65,10 @@ class ScrobblableMixin(TimeStampedModel):
     class Meta:
         abstract = True
 
+    @classmethod
+    def is_long_play_media(cls) -> bool:
+        return False
+
     def scrobble_for_user(
         self,
         user_id,
@@ -136,6 +140,15 @@ class LongPlayScrobblableMixin(ScrobblableMixin):
     class Meta:
         abstract = True
 
+    @classmethod
+    def is_long_play_media(cls) -> bool:
+        return True
+
+    def is_complete(self) -> bool:
+        if self.log:
+            return bool(self.log.get("long_play_complete", None))
+        return False
+
     def get_longplay_finish_url(self):
         return reverse("scrobbles:longplay-finish", kwargs={"uuid": self.uuid})
 

vrobbler/apps/scrobbles/models.py

@@ -738,7 +738,7 @@ class Scrobble(TimeStampedModel):
             log_dict = {}
 
         try:
-            return logdata_cls.from_dict(log_dict)
+            return logdata_cls(**log_dict)
         except ParseError as e:
             logger.warning(
                 "Could not parse log data",
@@ -749,6 +749,8 @@ class Scrobble(TimeStampedModel):
                 },
             )
             return logdata_cls()
+        except TypeError as e:
+            return logdata_cls()
 
     def redirect_url(self, user_id) -> str:
         user = User.objects.filter(id=user_id).first()
@@ -1065,8 +1067,7 @@ class Scrobble(TimeStampedModel):
         return media_obj
 
     def __str__(self):
-        timestamp = self.timestamp.strftime("%Y-%m-%d")
+        return f"Scrobble of {self.media_obj} ({self.timestamp})"
-        return f"Scrobble of {self.media_obj} ({timestamp})"
 
     def calc_reading_duration(self) -> int:
         duration = 0

vrobbler/apps/scrobbles/scrobblers.py

@@ -411,7 +411,7 @@ def email_scrobble_board_game(
                     except IndexError:
                         second = 0
 
-            log_data["details"] = play_dict.get("comments")
+            log_data["notes"] = [play_dict.get("comments")]
         log_data["expansion_ids"] = []
         try:
             base_game = base_games[play_dict.get("gameRefId")]
@@ -587,9 +587,9 @@ def todoist_scrobble_update_task(
         )
         return
 
-    existing_notes = scrobble.log.get("notes", {})
+    if not scrobble.log.get("notes"):
-    existing_notes[todoist_note.get("todoist_id")] = todoist_note.get("notes")
+        scrobble.log["notes"] = []
-    scrobble.log["notes"] = existing_notes
+    scrobble.log["notes"].append(todoist_note.get("notes"))
     scrobble.save(update_fields=["log"])
     logger.info(
         "[todoist_scrobble_update_task] todoist note added",
@@ -615,7 +615,7 @@ def todoist_scrobble_task(
     )
     task = Task.find_or_create(title)
 
-    timestamp = pendulum.parse(todoist_task.get("updated_at", timezone.now()))
+    timestamp = pendulum.parse(todoist_task.pop("updated_at", timezone.now()))
     in_progress_scrobble = Scrobble.objects.filter(
         user_id=user_id,
         in_progress=True,
@@ -657,8 +657,12 @@ def todoist_scrobble_task(
         )
         return todoist_scrobble_task_finish(todoist_task, user_id, timestamp)
 
-    # Default to create new scrobble "if not in_progress_scrobble and in_progress_in_todoist"
+    todoist_task["title"] = todoist_task.pop("description")
-    # TODO Should use updated_at from TOdoist, but parsing isn't working
+    todoist_task["description"] = todoist_task.pop("details")
+    todoist_task["labels"] = todoist_task.pop("todoist_label_list", [])
+    todoist_task.pop("todoist_type")
+    todoist_task.pop("todoist_event")
+
     scrobble_dict = {
         "user_id": user_id,
         "timestamp": timestamp,
@@ -686,8 +690,8 @@ def emacs_scrobble_update_task(
     scrobble = Scrobble.objects.filter(
         in_progress=True,
         user_id=user_id,
-        log__source_id=emacs_id,
+        log__orgmode_id=emacs_id,
-        log__source="orgmode",
+        source="Org-mode",
     ).first()
 
     if not scrobble:
@@ -736,18 +740,18 @@ def emacs_scrobble_task(
     stopped: bool = False,
     user_context_list: list[str] = [],
 ) -> Scrobble | None:
-    source_id = task_data.get("source_id")
+    orgmode_id = task_data.get("source_id")
     title = get_title_from_labels(
         task_data.get("labels", []), user_context_list
     )
 
     task = Task.find_or_create(title)
 
-    timestamp = pendulum.parse(task_data.get("updated_at", timezone.now()))
+    timestamp = pendulum.parse(task_data.pop("updated_at", timezone.now()))
     in_progress_scrobble = Scrobble.objects.filter(
         user_id=user_id,
         in_progress=True,
-        log__source_id=source_id,
+        log__orgmode_id=orgmode_id,
         log__source="orgmode",
         task=task,
     ).last()
@@ -756,7 +760,7 @@ def emacs_scrobble_task(
         logger.info(
             "[emacs_scrobble_task] cannot stop already stopped task",
             extra={
-                "emacs_id": source_id,
+                "orgmode_id": orgmode_id,
             },
         )
         return
@@ -765,7 +769,7 @@ def emacs_scrobble_task(
         logger.info(
             "[emacs_scrobble_task] cannot start already started task",
             extra={
-                "emacs_id": source_id,
+                "ormode_id": orgmode_id,
             },
         )
         return in_progress_scrobble
@@ -775,7 +779,7 @@ def emacs_scrobble_task(
         logger.info(
             "[emacs_scrobble_task] finishing",
             extra={
-                "emacs_id": source_id,
+                "orgmode_id": orgmode_id,
             },
         )
         in_progress_scrobble.stop(timestamp=timestamp, force_finish=True)
@@ -786,11 +790,17 @@ def emacs_scrobble_task(
 
     notes = task_data.pop("notes")
     if notes:
-        task_data["notes"] = []
+        task_data["notes"] = [note.get("content") for note in notes]
-        for note in notes:
+    task_data["title"] = task_data.pop("description")
-            task_data["notes"].append(
+    task_data["description"] = task_data.pop("body")
-                {note.get("timestamp"): note.get("content")}
+    task_data["labels"] = task_data.pop("labels")
-            )
+
+    task_data["orgmode_id"] = task_data.pop("source_id")
+    task_data["orgmode_state"] = task_data.pop("state")
+    task_data["orgmode_properties"] = task_data.pop("properties")
+    task_data["orgmode_drawers"] = task_data.pop("drawers")
+    task_data["orgmode_timestamps"] = task_data.pop("timestamps")
+    task_data.pop("source")
 
     scrobble_dict = {
         "user_id": user_id,

vrobbler/apps/tasks/models.py

@@ -5,7 +5,7 @@ from typing import Optional
 from django.apps import apps
 from django.db import models
 from django.urls import reverse
-from scrobbles.dataclasses import JSONDataclass
+from scrobbles.dataclasses import BaseLogData, JSONDataclass
 from scrobbles.mixins import LongPlayScrobblableMixin, ScrobblableConstants
 
 BNULL = {"blank": True, "null": True}
@@ -14,27 +14,18 @@ TODOIST_TASK_URL = "https://app.todoist.com/app/task/{id}"
 
 
 @dataclass
-class TaskLogData(JSONDataclass):
+class TaskLogData(BaseLogData):
-    description: Optional[str] = None
     title: Optional[str] = None
-    project: Optional[str] = None
+    labels: Optional[list[str]] = None
-    notes: Optional[dict] = None
+
-    updated_at: Optional[str] = None
+    orgmode_id: Optional[str] = None
+    orgmode_state: Optional[str] = None
+    orgmode_properties: Optional[dict] = None
+    orgmode_drawers: Optional[list] = None
+    orgmode_timestamps: Optional[list] = None
+
     todoist_id: Optional[str] = None
-    todoist_event: Optional[str] = None
-    todoist_type: Optional[str] = None
-    todoist_type: Optional[str] = None
-    todoist_label_list: Optional[list] = None
     todoist_project_id: Optional[str] = None
-    body: Optional[str] = None
-    state: Optional[str] = None
-    labels: Optional[str] = None
-    properties: Optional[list] = None
-    drawers: Optional[list] = None
-    source: Optional[str] = None
-    source_id: Optional[str] = None
-    timestamps: Optional[list] = None
-    details: Optional[str] = None
 
     def notes_as_str(self) -> str:
         """Return formatted notes with line breaks and no keys"""

vrobbler/apps/tasks/utils.py

@@ -1,22 +1,106 @@
 import logging
+from datetime import timedelta
 
 from django.conf import settings
+from scrobbles.models import Scrobble
 
 logger = logging.getLogger(__name__)
 
-def get_title_from_labels(labels: list[str], user_context_labels: list[str] = []) -> str:
+
+def get_title_from_labels(
+    labels: list[str], user_context_labels: list[str] = []
+) -> str:
     title = "Unknown"
-    task_context_labels: list = user_context_labels or settings.DEFAULT_TASK_CONTEXT_TAG_LIST
     for label in labels:
-        # TODO We may also want to take a user list of labels instead
         label = label.capitalize()
-        if label in task_context_labels:
+        if label in user_context_labels:
             title = label
             continue
 
     if title == "Unknown":
         logger.warning(
             "Missing a configured title context for task",
-            extra={"labels": labels, "task_context_labels": task_context_labels},
+            extra={
+                "labels": labels,
+                "user_context_labels": user_context_labels,
+            },
         )
     return title
+
+
+def convert_old_orgmode_log_to_new(commit=False):
+    scrobbles = Scrobble.objects.filter(
+        source="Org-mode", log__has_key="drawers"
+    )
+    for scrobble in scrobbles:
+        scrobble.log["title"] = scrobble.log.pop("description")
+        scrobble.log["description"] = scrobble.log.pop("details")
+
+        scrobble.log["orgmode_body"] = scrobble.log.pop("body")
+        scrobble.log["orgmode_state"] = scrobble.log.pop("state")
+        scrobble.log["orgmode_properties"] = scrobble.log.pop("properties")
+        scrobble.log["orgmode_drawers"] = scrobble.log.pop("drawers")
+        scrobble.log["orgmode_timestamps"] = scrobble.log.pop("timestamps")
+        scrobble.log["orgmode_id"] = scrobble.log.pop("source_id")
+
+        if commit:
+            scrobble.save(update_fields=["log"])
+    print(f"Updated {scrobbles.count()} orgmode tasks logs")
+
+
+def convert_old_todoist_log_to_new(commit=False):
+    scrobbles = Scrobble.objects.filter(
+        source="Todoist", log__has_key="todoist_type"
+    )
+    for scrobble in scrobbles:
+        scrobble.log["title"] = scrobble.log.pop("description")
+        scrobble.log["description"] = scrobble.log.pop("details")
+        scrobble.log["todoist_id"] = scrobble.log.pop("source_id")
+        scrobble.log["labels"] = scrobble.log.pop("todoist_label_list")
+
+        scrobble.log.pop("todoist_type")
+        scrobble.log.pop("todoist_event")
+
+        print(f"Updating scrobble {scrobble.id}")
+        if commit:
+            scrobble.save(update_fields=["log"])
+    print(f"Converted {scrobbles.count()} todoist tasks logs")
+
+
+def convert_notes_to_dict(commit=False):
+    scrobbles = Scrobble.objects.filter(log__notes__isnull=False)
+    count = 0
+    for scrobble in scrobbles:
+        if isinstance(scrobble.log, str):
+            print(f"Converting {scrobble} string note to dict")
+            if scrobble.log.get("notes") == "":
+                scrobble.log.pop("notes")
+            key = str(int(scrobble.timestamp.timestamp()))
+            notes = scrobble.log.pop("notes")
+            scrobble.log = {}
+            scrobble.log["notes"] = {key: notes}
+            count += 1
+
+        if isinstance(scrobble.log.get("notes"), list):
+            note_list = scrobble.log.pop("notes")
+            if all(isinstance(item, dict) for item in note_list):
+                scrobble.log["notes"] = [
+                    value for d in note_list for value in d.values()
+                ]
+            count += 1
+        if commit:
+            scrobble.save(update_fields=["log"])
+    print(f"Updated {count} todoist tasks scrobbles")
+
+
+def convert_old_boardgame_log_to_new(commit=False):
+    scrobbles = Scrobble.objects.filter(
+        board_game__isnull=False, log__has_key="notes"
+    )
+    for scrobble in scrobbles:
+        if isinstance(scrobble.log.get("notes"), str):
+            scrobble.log["notes"] = [scrobble.log.pop("notes")]
+
+        if commit:
+            scrobble.save(update_fields=["log"])
+    print(f"Updated {scrobbles.count()} board game scrobbles")