secstate/vrobbler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[scrobbles] Start cleaning up logdata

Browse Source
This commit is contained in:
Colin Powell
2025-08-05 02:01:31 -04:00
parent 06e075553a
commit 278cab32ea
9 changed files with 191 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
PROJECT.org
View File

@ -443,6 +443,10 @@ it's annoying.
** TODO [#C] Allow users to see tasks on calendar view :vrobbler:personal:project:templates:feature:
https://codepen.io/oliviale/pen/QYqybo
** TODO [#C] Come up with a possible flow using WebDAV and super-productivity for tasks :personal:feature:project:vrobbler:tasks:
** DONE Fix long play scrobbles to provide better data :vrobbler:feature:scrobbles:longplay:personal:project:
:PROPERTIES:
:ID: 99f6bd77-dc8f-6ed1-0321-32a52c944264
:END:
* Version 19.0
** DONE Add periodic check for mood :vrobbler:feature:moods:personal:project:
:PROPERTIES:

20
vrobbler/apps/music/models.py
View File

@ -1,5 +1,6 @@
import logging
from typing import Dict, Optional
from dataclasses import dataclass
from typing import Optional
from uuid import uuid4
import musicbrainzngs
@ -15,24 +16,26 @@ from imagekit.processors import ResizeToFit
from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic
from music.bandcamp import get_bandcamp_slug
from music.musicbrainz import (
get_album_metadata,
get_album_metadata_with_artist,
get_artist_metadata_extended,
get_recording_mbid_exact,
get_track_metadata_with_artist,
lookup_album_dict_from_mb,
lookup_album_from_mb,
lookup_track_from_mb,
lookup_artist_from_mb,
)
from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb
from music.utils import clean_artist_name
from scrobbles.dataclasses import BaseLogData
from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
logger = logging.getLogger(__name__)
BNULL = {"blank": True, "null": True}
@dataclass
class TrackLogData(BaseLogData):
mopidy_source: Optional[str] = None
rockbox_info: Optional[str] = None
rating: Optional[int] = None
class Artist(TimeStampedModel):
"""Represents a music artist.
@ -605,6 +608,9 @@ class Track(ScrobblableMixin):
def __str__(self):
return f"{self.title} by {self.artist}"
def logdata_cls(self):
return TrackLogData
@property
def primary_album(self):
if self.album:

3
vrobbler/apps/people/models.py
View File

@ -15,3 +15,6 @@ class Person(TimeStampedModel):
bgg_username = models.CharField(max_length=100, **BNULL)
lichess_username = models.CharField(max_length=100, **BNULL)
bio = models.TextField(**BNULL)
def __str__(self):
return self.name

25
vrobbler/apps/scrobbles/management/commands/convert_task_log_data.py Normal file
View File

@ -0,0 +1,25 @@
from django.core.management.base import BaseCommand
from vrobbler.apps.tasks.utils import (
convert_notes_to_dict,
convert_old_boardgame_log_to_new,
convert_old_orgmode_log_to_new,
)
class Command(BaseCommand):
def add_arguments(self, parser):
parser.add_argument(
"--commit",
action="store_true",
help="Commit changes",
)
def handle(self, *args, **options):
commit = False
if options["commit"]:
commit = True
else:
print("No changes will be saved, use --commit to save")
convert_old_orgmode_log_to_new(commit)
convert_notes_to_dict(commit)
convert_old_boardgame_log_to_new(commit)

13
vrobbler/apps/scrobbles/mixins.py
View File

@ -65,6 +65,10 @@ class ScrobblableMixin(TimeStampedModel):
class Meta:
abstract = True
@classmethod
def is_long_play_media(cls) -> bool:
return False
def scrobble_for_user(
self,
user_id,
@ -136,6 +140,15 @@ class LongPlayScrobblableMixin(ScrobblableMixin):
class Meta:
abstract = True
@classmethod
def is_long_play_media(cls) -> bool:
return True
def is_complete(self) -> bool:
if self.log:
return bool(self.log.get("long_play_complete", None))
return False
def get_longplay_finish_url(self):
return reverse("scrobbles:longplay-finish", kwargs={"uuid": self.uuid})

7
vrobbler/apps/scrobbles/models.py
View File

@ -738,7 +738,7 @@ class Scrobble(TimeStampedModel):
log_dict = {}
try:
return logdata_cls.from_dict(log_dict)
return logdata_cls(**log_dict)
except ParseError as e:
logger.warning(
"Could not parse log data",
@ -749,6 +749,8 @@ class Scrobble(TimeStampedModel):
},
)
return logdata_cls()
except TypeError as e:
return logdata_cls()
def redirect_url(self, user_id) -> str:
user = User.objects.filter(id=user_id).first()
@ -1065,8 +1067,7 @@ class Scrobble(TimeStampedModel):
return media_obj
def __str__(self):
timestamp = self.timestamp.strftime("%Y-%m-%d")
return f"Scrobble of {self.media_obj} ({timestamp})"
return f"Scrobble of {self.media_obj} ({self.timestamp})"
def calc_reading_duration(self) -> int:
duration = 0

50
vrobbler/apps/scrobbles/scrobblers.py
View File

@ -411,7 +411,7 @@ def email_scrobble_board_game(
except IndexError:
second = 0
log_data["details"] = play_dict.get("comments")
log_data["notes"] = [play_dict.get("comments")]
log_data["expansion_ids"] = []
try:
base_game = base_games[play_dict.get("gameRefId")]
@ -587,9 +587,9 @@ def todoist_scrobble_update_task(
)
return
existing_notes = scrobble.log.get("notes", {})
existing_notes[todoist_note.get("todoist_id")] = todoist_note.get("notes")
scrobble.log["notes"] = existing_notes
if not scrobble.log.get("notes"):
scrobble.log["notes"] = []
scrobble.log["notes"].append(todoist_note.get("notes"))
scrobble.save(update_fields=["log"])
logger.info(
"[todoist_scrobble_update_task] todoist note added",
@ -615,7 +615,7 @@ def todoist_scrobble_task(
)
task = Task.find_or_create(title)
timestamp = pendulum.parse(todoist_task.get("updated_at", timezone.now()))
timestamp = pendulum.parse(todoist_task.pop("updated_at", timezone.now()))
in_progress_scrobble = Scrobble.objects.filter(
user_id=user_id,
in_progress=True,
@ -657,8 +657,12 @@ def todoist_scrobble_task(
)
return todoist_scrobble_task_finish(todoist_task, user_id, timestamp)
# Default to create new scrobble "if not in_progress_scrobble and in_progress_in_todoist"
# TODO Should use updated_at from TOdoist, but parsing isn't working
todoist_task["title"] = todoist_task.pop("description")
todoist_task["description"] = todoist_task.pop("details")
todoist_task["labels"] = todoist_task.pop("todoist_label_list", [])
todoist_task.pop("todoist_type")
todoist_task.pop("todoist_event")
scrobble_dict = {
"user_id": user_id,
"timestamp": timestamp,
@ -686,8 +690,8 @@ def emacs_scrobble_update_task(
scrobble = Scrobble.objects.filter(
in_progress=True,
user_id=user_id,
log__source_id=emacs_id,
log__source="orgmode",
log__orgmode_id=emacs_id,
source="Org-mode",
).first()
if not scrobble:
@ -736,18 +740,18 @@ def emacs_scrobble_task(
stopped: bool = False,
user_context_list: list[str] = [],
) -> Scrobble | None:
source_id = task_data.get("source_id")
orgmode_id = task_data.get("source_id")
title = get_title_from_labels(
task_data.get("labels", []), user_context_list
)
task = Task.find_or_create(title)
timestamp = pendulum.parse(task_data.get("updated_at", timezone.now()))
timestamp = pendulum.parse(task_data.pop("updated_at", timezone.now()))
in_progress_scrobble = Scrobble.objects.filter(
user_id=user_id,
in_progress=True,
log__source_id=source_id,
log__orgmode_id=orgmode_id,
log__source="orgmode",
task=task,
).last()
@ -756,7 +760,7 @@ def emacs_scrobble_task(
logger.info(
"[emacs_scrobble_task] cannot stop already stopped task",
extra={
"emacs_id": source_id,
"orgmode_id": orgmode_id,
},
)
return
@ -765,7 +769,7 @@ def emacs_scrobble_task(
logger.info(
"[emacs_scrobble_task] cannot start already started task",
extra={
"emacs_id": source_id,
"ormode_id": orgmode_id,
},
)
return in_progress_scrobble
@ -775,7 +779,7 @@ def emacs_scrobble_task(
logger.info(
"[emacs_scrobble_task] finishing",
extra={
"emacs_id": source_id,
"orgmode_id": orgmode_id,
},
)
in_progress_scrobble.stop(timestamp=timestamp, force_finish=True)
@ -786,11 +790,17 @@ def emacs_scrobble_task(
notes = task_data.pop("notes")
if notes:
task_data["notes"] = []
for note in notes:
task_data["notes"].append(
{note.get("timestamp"): note.get("content")}
)
task_data["notes"] = [note.get("content") for note in notes]
task_data["title"] = task_data.pop("description")
task_data["description"] = task_data.pop("body")
task_data["labels"] = task_data.pop("labels")
task_data["orgmode_id"] = task_data.pop("source_id")
task_data["orgmode_state"] = task_data.pop("state")
task_data["orgmode_properties"] = task_data.pop("properties")
task_data["orgmode_drawers"] = task_data.pop("drawers")
task_data["orgmode_timestamps"] = task_data.pop("timestamps")
task_data.pop("source")
scrobble_dict = {
"user_id": user_id,

29
vrobbler/apps/tasks/models.py
View File

@ -5,7 +5,7 @@ from typing import Optional
from django.apps import apps
from django.db import models
from django.urls import reverse
from scrobbles.dataclasses import JSONDataclass
from scrobbles.dataclasses import BaseLogData, JSONDataclass
from scrobbles.mixins import LongPlayScrobblableMixin, ScrobblableConstants
BNULL = {"blank": True, "null": True}
@ -14,27 +14,18 @@ TODOIST_TASK_URL = "https://app.todoist.com/app/task/{id}"
@dataclass
class TaskLogData(JSONDataclass):
description: Optional[str] = None
class TaskLogData(BaseLogData):
title: Optional[str] = None
project: Optional[str] = None
notes: Optional[dict] = None
updated_at: Optional[str] = None
labels: Optional[list[str]] = None
orgmode_id: Optional[str] = None
orgmode_state: Optional[str] = None
orgmode_properties: Optional[dict] = None
orgmode_drawers: Optional[list] = None
orgmode_timestamps: Optional[list] = None
todoist_id: Optional[str] = None
todoist_event: Optional[str] = None
todoist_type: Optional[str] = None
todoist_type: Optional[str] = None
todoist_label_list: Optional[list] = None
todoist_project_id: Optional[str] = None
body: Optional[str] = None
state: Optional[str] = None
labels: Optional[str] = None
properties: Optional[list] = None
drawers: Optional[list] = None
source: Optional[str] = None
source_id: Optional[str] = None
timestamps: Optional[list] = None
details: Optional[str] = None
def notes_as_str(self) -> str:
"""Return formatted notes with line breaks and no keys"""

94
vrobbler/apps/tasks/utils.py
View File

@ -1,22 +1,106 @@
import logging
from datetime import timedelta
from django.conf import settings
from scrobbles.models import Scrobble
logger = logging.getLogger(__name__)
def get_title_from_labels(labels: list[str], user_context_labels: list[str] = []) -> str:
def get_title_from_labels(
labels: list[str], user_context_labels: list[str] = []
) -> str:
title = "Unknown"
task_context_labels: list = user_context_labels or settings.DEFAULT_TASK_CONTEXT_TAG_LIST
for label in labels:
# TODO We may also want to take a user list of labels instead
label = label.capitalize()
if label in task_context_labels:
if label in user_context_labels:
title = label
continue
if title == "Unknown":
logger.warning(
"Missing a configured title context for task",
extra={"labels": labels, "task_context_labels": task_context_labels},
extra={
"labels": labels,
"user_context_labels": user_context_labels,
},
)
return title
def convert_old_orgmode_log_to_new(commit=False):
scrobbles = Scrobble.objects.filter(
source="Org-mode", log__has_key="drawers"
)
for scrobble in scrobbles:
scrobble.log["title"] = scrobble.log.pop("description")
scrobble.log["description"] = scrobble.log.pop("details")
scrobble.log["orgmode_body"] = scrobble.log.pop("body")
scrobble.log["orgmode_state"] = scrobble.log.pop("state")
scrobble.log["orgmode_properties"] = scrobble.log.pop("properties")
scrobble.log["orgmode_drawers"] = scrobble.log.pop("drawers")
scrobble.log["orgmode_timestamps"] = scrobble.log.pop("timestamps")
scrobble.log["orgmode_id"] = scrobble.log.pop("source_id")
if commit:
scrobble.save(update_fields=["log"])
print(f"Updated {scrobbles.count()} orgmode tasks logs")
def convert_old_todoist_log_to_new(commit=False):
scrobbles = Scrobble.objects.filter(
source="Todoist", log__has_key="todoist_type"
)
for scrobble in scrobbles:
scrobble.log["title"] = scrobble.log.pop("description")
scrobble.log["description"] = scrobble.log.pop("details")
scrobble.log["todoist_id"] = scrobble.log.pop("source_id")
scrobble.log["labels"] = scrobble.log.pop("todoist_label_list")
scrobble.log.pop("todoist_type")
scrobble.log.pop("todoist_event")
print(f"Updating scrobble {scrobble.id}")
if commit:
scrobble.save(update_fields=["log"])
print(f"Converted {scrobbles.count()} todoist tasks logs")
def convert_notes_to_dict(commit=False):
scrobbles = Scrobble.objects.filter(log__notes__isnull=False)
count = 0
for scrobble in scrobbles:
if isinstance(scrobble.log, str):
print(f"Converting {scrobble} string note to dict")
if scrobble.log.get("notes") == "":
scrobble.log.pop("notes")
key = str(int(scrobble.timestamp.timestamp()))
notes = scrobble.log.pop("notes")
scrobble.log = {}
scrobble.log["notes"] = {key: notes}
count += 1
if isinstance(scrobble.log.get("notes"), list):
note_list = scrobble.log.pop("notes")
if all(isinstance(item, dict) for item in note_list):
scrobble.log["notes"] = [
value for d in note_list for value in d.values()
]
count += 1
if commit:
scrobble.save(update_fields=["log"])
print(f"Updated {count} todoist tasks scrobbles")
def convert_old_boardgame_log_to_new(commit=False):
scrobbles = Scrobble.objects.filter(
board_game__isnull=False, log__has_key="notes"
)
for scrobble in scrobbles:
if isinstance(scrobble.log.get("notes"), str):
scrobble.log["notes"] = [scrobble.log.pop("notes")]
if commit:
scrobble.save(update_fields=["log"])
print(f"Updated {scrobbles.count()} board game scrobbles")