From dec7a79509d62763a0fcec9e86c710b4ef1a1a1b Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Fri, 5 Jun 2026 19:31:06 -0400 Subject: [PATCH] [scrobbles] Add basic sentiment analysis --- PROJECT.org | 23 +++++-- poetry.lock | 17 ++++- pyproject.toml | 1 + .../commands/backfill_scrobble_sentiment.py | 68 +++++++++++++++++++ vrobbler/apps/scrobbles/models.py | 8 ++- vrobbler/apps/scrobbles/tasks.py | 25 +++++++ vrobbler/apps/scrobbles/utils.py | 29 ++++++++ vrobbler/settings.py | 4 ++ .../templates/scrobbles/scrobble_detail.html | 21 ++++++ 9 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 vrobbler/apps/scrobbles/management/commands/backfill_scrobble_sentiment.py diff --git a/PROJECT.org b/PROJECT.org index 87a673a..498b4f2 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -93,11 +93,7 @@ fetching and simple saving. :LOGBOOK: CLOCK: [2025-07-09 Wed 09:55]--[2025-07-09 Wed 10:15] => 0:20 :END: -* Backlog [0/13] :vrobbler:project:personal: -** TODO [#C] Add sentiment parsing for Scrobbles with notes :vrobbler:project:scrobbles:sentiment: -:PROPERTIES: -:ID: 37781d6a-f3b0-48b2-bf98-33c2c791cf85 -:END: +* Backlog [1/13] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :vrobbler:personal:bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -486,7 +482,7 @@ whatever time KoReader reports, we need to know, given the date and the user profile's historic timezone, how many hours to adjust the KoReader time to get to GMT to save it in the database. -** TODO [#A] Make IMAP and WebDAV configurable :webdav:feature:imap:importers: +** TODO [#B] Make IMAP and WebDAV configurable :webdav:feature:imap:importers: :PROPERTIES: :ID: b1426d92-2feb-4d15-9738-d5b7b0594f96 :END: @@ -509,6 +505,21 @@ needed import celery task. This is how the WebDAV celery task currently works. This would also be an opporunity to clean up the code around WebDAV imports and make them more re-usable for other import services. +** DONE [#C] Add sentiment parsing for Scrobbles with notes :scrobbles:sentiment: +:PROPERTIES: +:ID: 37781d6a-f3b0-48b2-bf98-33c2c791cf85 +:END: + +*** Description + +Not sure how useful this would be, but I wonder if we can add a `sentiment` JSONField +on each scrobble that can store the output of VADER over the notes in a scrobble with notes. + +I'm not sure that the value prop here is worth the storage and processing time. + +But if we do add it, it should be a process that scans for scrobbles with both notes and no +sentiment field value (unless --overwrite is used) and just run periodically. + * Version 45.1 [1/1] ** DONE [#B] Mopidy favorites or monthly playlist adds should look at all scrobbles :bug:mopidy:favorites:tracks: :PROPERTIES: diff --git a/poetry.lock b/poetry.lock index f335328..e76bdc1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5439,6 +5439,21 @@ brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and p secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "vadersentiment" +version = "3.3.2" +description = "VADER Sentiment Analysis. VADER (Valence Aware Dictionary and sEntiment Reasoner) is a lexicon and rule-based sentiment analysis tool that is specifically attuned to sentiments expressed in social media, and works well on texts from other domains." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "vaderSentiment-3.3.2-py2.py3-none-any.whl", hash = "sha256:3bf1d243b98b1afad575b9f22bc2cb1e212b94ff89ca74f8a23a588d024ea311"}, + {file = "vaderSentiment-3.3.2.tar.gz", hash = "sha256:5d7c06e027fc8b99238edb0d53d970cf97066ef97654009890b83703849632f9"}, +] + +[package.dependencies] +requests = "*" + [[package]] name = "vine" version = "5.1.0" @@ -6005,4 +6020,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = ">=3.11,<3.15" -content-hash = "bd3f14a9cfce403db426af98774f1e3c41b97283aa43f4bd80f84594ee0dd726" +content-hash = "78ba52d0e6ea492efceb14fcd42ace25abfb66d42c3aff28f2fe1a31a9aa03b5" diff --git a/pyproject.toml b/pyproject.toml index 2800a6d..531e5d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ recipe-scrapers = "^15.11.0" gpxpy = "^1.6.2" fitparse = "^1.2.0" lxml = ">=5.5.0" +vaderSentiment = "^3.3.2" [tool.poetry.group.test] optional = true diff --git a/vrobbler/apps/scrobbles/management/commands/backfill_scrobble_sentiment.py b/vrobbler/apps/scrobbles/management/commands/backfill_scrobble_sentiment.py new file mode 100644 index 0000000..377f877 --- /dev/null +++ b/vrobbler/apps/scrobbles/management/commands/backfill_scrobble_sentiment.py @@ -0,0 +1,68 @@ +from django.core.management.base import BaseCommand +from django.db import models + +from scrobbles.models import Scrobble +from scrobbles.utils import analyze_scrobble_sentiment + + +class Command(BaseCommand): + help = "Backfill VADER sentiment analysis for scrobbles with notes" + + def add_arguments(self, parser): + parser.add_argument( + "--commit", + action="store_true", + help="Actually update scrobble logs with sentiment data", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Re-analyze scrobbles that already have sentiment data", + ) + + def handle(self, *args, **options): + commit = options["commit"] + overwrite = options["overwrite"] + + qs = Scrobble.objects.filter( + ~models.Q(log__notes__isnull=True) + & ~models.Q(log__notes=[]) + & ~models.Q(log__notes={}) + ) + if not overwrite: + qs = qs.filter(log__sentiment__isnull=True) + + total = qs.count() + analyzed_count = 0 + skipped_count = 0 + + self.stdout.write(f"Found {total} scrobbles to process") + + for scrobble in qs.iterator(): + if commit: + analyzed = analyze_scrobble_sentiment(scrobble, overwrite=overwrite) + else: + notes_str = "" + if scrobble.logdata: + notes_str = scrobble.logdata.notes_as_str() + analyzed = bool(notes_str) + + if analyzed: + analyzed_count += 1 + if commit: + scores = (scrobble.log or {}).get("sentiment", {}) + self.stdout.write( + f" Updated scrobble {scrobble.id}: compound={scores.get('compound', 'N/A')}" + ) + else: + self.stdout.write( + f" [DRY RUN] Would analyze scrobble {scrobble.id}" + ) + else: + skipped_count += 1 + + self.stdout.write( + f"\nAnalyzed {analyzed_count} scrobbles, skipped {skipped_count}" + ) + if not commit: + self.stdout.write("Run with --commit to persist changes") diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 8b9eca5..35ea184 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -882,8 +882,14 @@ class Scrobble(TimeStampedModel): logger.warning("Log data could not be loaded", e) return logdata_cls() + # Strip log-only keys (stored in JSONField but not part of LogData dataclass) + logdata_kwargs = { + k: v for k, v in log_dict.items() + if k in logdata_cls.__dataclass_fields__ + } + try: - return logdata_cls(**log_dict) + return logdata_cls(**logdata_kwargs) except ParseError as e: logger.warning( "Could not parse log data", diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index 37abca3..bbc4fe7 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -12,6 +12,7 @@ from charts.utils import ( from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model +from django.db import models from django.utils import timezone logger = logging.getLogger(__name__) @@ -544,6 +545,30 @@ def send_mood_checkin(): send_mood_checkin_reminders() +@shared_task +def backfill_scrobble_sentiment(): + """Backfill VADER sentiment for scrobbles with notes (replaces @hourly cron).""" + from scrobbles.models import Scrobble + from scrobbles.utils import analyze_scrobble_sentiment + + qs = Scrobble.objects.filter( + models.Q(log__notes__isnull=False) + & ~models.Q(log__notes=[]) + & ~models.Q(log__notes={}) + & models.Q(log__sentiment__isnull=True) + ) + + count = 0 + for scrobble in qs.iterator(): + if analyze_scrobble_sentiment(scrobble): + count += 1 + + logger.info( + "Backfilled sentiment for %d scrobbles", + count, + ) + + @shared_task def add_favorite_to_mopidy_playlist(favorite_id): from scrobbles.models import FavoriteMedia diff --git a/vrobbler/apps/scrobbles/utils.py b/vrobbler/apps/scrobbles/utils.py index 5b7a949..21b3384 100644 --- a/vrobbler/apps/scrobbles/utils.py +++ b/vrobbler/apps/scrobbles/utils.py @@ -802,3 +802,32 @@ def tokenize_title_to_tags(title: str) -> list[str]: w.lower() for w in cleaned.split() if w.lower() not in STOPWORDS and len(w) > 2 ] return words + + +def analyze_scrobble_sentiment(scrobble, overwrite=False) -> bool: + """Run VADER sentiment analysis on a scrobble's notes. + + Stores result in log["sentiment"] as a dict with keys: + neg, neu, pos, compound. + + Returns True if analyzed, False if skipped (no notes or already done). + """ + from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer + + log = scrobble.log or {} + if not overwrite and log.get("sentiment") is not None: + return False + + notes_str = "" + if scrobble.logdata: + notes_str = scrobble.logdata.notes_as_str() + if not notes_str: + return False + + analyzer = SentimentIntensityAnalyzer() + scores = analyzer.polarity_scores(notes_str) + + log["sentiment"] = scores + scrobble.log = log + scrobble.save(update_fields=["log"]) + return True diff --git a/vrobbler/settings.py b/vrobbler/settings.py index f394fe2..cdeb8a0 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -163,6 +163,10 @@ CELERY_BEAT_SCHEDULE = { "task": "scrobbles.tasks.send_mood_checkin", "schedule": crontab(hour="*/4", minute=0), }, + "backfill-scrobble-sentiment": { + "task": "scrobbles.tasks.backfill_scrobble_sentiment", + "schedule": crontab(minute="0"), + }, } INSTALLED_APPS = [ diff --git a/vrobbler/templates/scrobbles/scrobble_detail.html b/vrobbler/templates/scrobbles/scrobble_detail.html index 3a821f1..86747c4 100644 --- a/vrobbler/templates/scrobbles/scrobble_detail.html +++ b/vrobbler/templates/scrobbles/scrobble_detail.html @@ -154,6 +154,20 @@ {% if notes_html %}

Notes

+ + {% if sentiment.compound >= 0.5 %}Positive + {% elif sentiment.compound >= 0.05 %}Slightly positive + {% elif sentiment.compound > -0.05 %}Neutral + {% elif sentiment.compound > -0.5 %}Slightly negative + {% else %}Negative + {% endif %} +
{{ notes_html|safe }}
@@ -161,6 +175,13 @@ {% endif %} {% endwith %} +{% with sentiment=object.log.sentiment %} +{% if sentiment %} +
+
+{% endif %} +{% endwith %} + {% if object.logdata.avg_seconds_per_page %}

Rate: {{object.logdata.avg_seconds_per_page}}s per page

{% endif %}