[celery] Queue-based task prioritization (672c81bf)

.drone.yml

@@ -37,7 +37,7 @@ steps:
         - pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git@main
         - vrobbler migrate
         - vrobbler collectstatic --noinput
-        - service vrobbler_celery restart && service vrobbler_celerybeat restart && service vrobbler restart
+        - service vrobbler_celery restart && service vrobbler_celery_priority restart && service vrobbler_celery_background restart && service vrobbler_celerybeat restart && service vrobbler restart
     when:
       ref:
         - refs/tags/*

.gitea/workflows/ci.yml

@@ -137,7 +137,7 @@ jobs:
             python3 -c "import vrobbler; print(f'vrobbler {vrobbler.__version__} installed OK')"
             vrobbler migrate
             vrobbler collectstatic --noinput
-            service vrobbler_celery restart && service vrobbler_celerybeat restart && service vrobbler restart
+            service vrobbler_celery restart && service vrobbler_celery_priority restart && service vrobbler_celery_background restart && service vrobbler_celerybeat restart && service vrobbler restart
 
       - name: Notify deploy success (ntfy)
         if: success()

CHANGELOG.org

@@ -117,6 +117,34 @@ Import gym sessions and routines from a Flexify backup.
   cover body-composition metrics on Task scrobbles and `bodyweight_kg` on
   Workout scrobbles.
 
+** DONE [#A] Fix celery task prioritization especially for agent sessions :celery:tasks:agents:
+:PROPERTIES:
+:ID:       672c81bf-bba9-a963-e8ac-52246d976cea
+:END:
+
+Split Celery work into four queues so time-sensitive tasks run promptly even
+when long batch jobs are queued:
+
+- `priority` — interactive/user-facing tasks (agent session prompts, Mopidy
+  queue/playlist adds, favorite toggles, reverse geocoding) get a dedicated
+  high-concurrency worker (`--concurrency=4 --prefetch-multiplier=1`).
+- `charts` — per-scrobble chart updates (`update_charts_for_timestamp`) stay on
+  their own queue, no longer blocked behind scheduled chart rebuilds (those
+  moved to `background`).
+- `background` — heavy/scheduled jobs (db backup, trend computation, sentiment
+  backfill, chart rebuilds, twitch VOD check, historical Last.fm dispatch) run
+  on a dedicated low-concurrency worker (`--concurrency=1`).
+- `default` — everything else (imports, notifications, archivebox pushes,
+  board-game expansion fetches). `CELERY_TASK_DEFAULT_QUEUE` is now set
+  explicitly to `default`; previously un-routed tasks were published to
+  Celery's default `celery` queue, which no worker consumed.
+
+Workers run as three FreeBSD daemons (`vrobbler_celery` on `default,charts`,
+plus new `vrobbler_celery_priority` and `vrobbler_celery_background` rc.d
+scripts with distinct nodenames). Procfile, justfile, Makefile, Drone and
+Gitea workflows updated accordingly.
+
+
 * Version 64.5 [1/1]
 ** DONE [#B] Clean up issues with org-mode notes :orgmode:notes:scrobbles:
 :PROPERTIES:

Makefile

@@ -1,5 +1,5 @@
 deploy:
-	ssh vrobbler.service "pip uninstall vrobbler && pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git && service vrobbler restart && service vrobbler_celery restart && service vrobbler_celerybeat restart && vrobbler migrate"
+	ssh vrobbler.service "pip uninstall vrobbler && pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git && service vrobbler restart && service vrobbler_celery restart && service vrobbler_celery_priority restart && service vrobbler_celery_background restart && service vrobbler_celerybeat restart && vrobbler migrate"
 logs:
 	ssh life.unbl.ink tail -n 100 -f /var/log/vrobbler.json
 test:

PROJECT.org

@@ -18,7 +18,7 @@ tasks, Todoist tasks, web pages I've read and trails I've hiked has turned out
 to be sometimes cathartic and sometimes functional as I try to remember when I
 did a thing.
 
-* Backlog [0/24] :vrobbler:project:personal:
+* Backlog [2/26] :vrobbler:project:personal:
 ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata:
 :PROPERTIES:
 :ID:       d3cce1a7-d540-4d66-bf66-e75378e4eac7
@@ -556,4 +556,32 @@ Added a `workouts` Django app so we can scrobble gym sessions.
 - Templates, admin, DRF viewsets (`exercises`, `workout-routines`), MCP tools,
   and `tests/workouts_tests/` (unit conversion, form round-trips, importer,
   imperial POST) included.
+** DONE [#A] Fix celery task prioritization especially for agent sessions :celery:tasks:agents:
+:PROPERTIES:
+:ID:       672c81bf-bba9-a963-e8ac-52246d976cea
+:END:
 
+*** Description
+
+Split Celery work into four queues so time-sensitive tasks run fast even when
+long batch jobs are queued up:
+
+- `priority` — interactive/user-facing tasks (agent session prompts, Mopidy
+  queue/playlist adds, favorite toggles, reverse geocoding) get a dedicated
+  high-concurrency worker with `--prefetch-multiplier=1`.
+- `charts` — per-scrobble chart updates stay on their own queue, no longer
+  blocked by scheduled chart rebuilds.
+- `background` — heavy/scheduled jobs (db backup, trends, sentiment backfill,
+  chart rebuilds, historical imports, twitch VOD check) run on a dedicated
+  low-concurrency worker so they can't starve the others.
+- `default` — everything else (imports, notifications, archivebox, expansions).
+
+Also set `CELERY_TASK_DEFAULT_QUEUE = "default"` — previously un-routed tasks
+were published to Celery's default `celery` queue, which no worker consumed.
+New rc.d scripts `vrobbler_celery_priority` and `vrobbler_celery_background`
+added alongside the existing `vrobbler_celery` (now `default,charts`).
+
+** DONE [#A] Fix lastfm rate limiting dropping scrobbles :importers:scrobbles:lastfm:
+:PROPERTIES:
+:ID:       e6718364-8f9e-d156-3a79-2e89b2ac697b
+:END:

Procfile

@@ -1,2 +1,4 @@
 web: python manage.py runserver 0.0.0.0:8014
 worker: celery -A vrobbler worker -Q default,charts -l DEBUG
+worker-priority: celery -A vrobbler worker -Q priority --concurrency=4 --prefetch-multiplier=1 -l DEBUG
+worker-background: celery -A vrobbler worker -Q background --concurrency=1 -l DEBUG

justfile

@@ -10,7 +10,13 @@ shell:
     poetry run python manage.py shell
 
 celery:
-    poetry run celery -A vrobbler worker -l info --concurrency=2 --pool=threads
+    poetry run celery -A vrobbler worker -Q default,charts -l info --concurrency=2 --pool=threads
+
+celery-priority:
+    poetry run celery -A vrobbler worker -Q priority -l info --concurrency=2 --pool=threads
+
+celery-background:
+    poetry run celery -A vrobbler worker -Q background -l info --concurrency=1 --pool=threads
 
 celery-beat:
     poetry run celery -A vrobbler beat -l info

rc.d/vrobbler_celery

@@ -29,9 +29,10 @@
 # is what `service vrobbler_celery stop` does) forwards the signal to celery,
 # which shuts the worker down gracefully.
 #
-# The queues match the old immortal/Procfile setup.  If you ever run more than
-# one worker instance, give each a unique nodename via -n (celery@%h is the
-# hostname default).
+# The default,charts worker handles on-demand tasks plus per-scrobble chart
+# updates.  Companion workers (vrobbler_celery_priority, vrobbler_celery_background)
+# consume the priority and background queues with distinct nodenames so they can
+# run side by side on one host.
 #
 . /etc/rc.subr
 
@@ -49,6 +50,6 @@ load_rc_config ${name}
 pidfile="/var/run/${name}.pid"
 
 command="/usr/sbin/daemon"
-command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_bin} -A vrobbler worker -Q ${vrobbler_celery_queues} -n celery@%h -l info --logfile ${vrobbler_log_dir}/${name}.log"
+command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_bin} -A vrobbler worker -Q ${vrobbler_celery_queues} -n default@%h -l info --logfile ${vrobbler_log_dir}/${name}.log"
 
 run_rc_command "$1"

rc.d/vrobbler_celery_background

@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# rc.d script for the vrobbler background Celery worker.
+#
+# PROVIDE: vrobbler_celery_background
+# REQUIRE: NETWORKING postgresql redis
+# KEYWORD: shutdown
+#
+# Installation (on the FreeBSD jail):
+#
+#   install -m 0555 rc.d/vrobbler_celery_background /usr/local/etc/rc.d/vrobbler_celery_background
+#   sysrc vrobbler_celery_background_enable="YES"
+#   service vrobbler_celery_background start
+#
+# Configuration (via sysrc or /etc/rc.conf):
+#
+#   vrobbler_celery_background_queues  - queues to consume  (default: background)
+#   vrobbler_celery_background_bin     - path to celery     (default: /usr/local/bin/celery)
+#   vrobbler_celery_background_flags   - extra daemon(8) flags
+#
+# Consumes the "background" queue where heavy, scheduled off-peak jobs are
+# routed (database backups, trend computation, sentiment backfill, chart
+# rebuilds, historical imports). Low concurrency keeps these CPU/I/O heavy jobs
+# from starving the other workers. The nodename is fixed to
+# background@<hostname> so it can coexist with the other workers on one host.
+#
+. /etc/rc.subr
+
+name="vrobbler_celery_background"
+rcvar="vrobbler_celery_background_enable"
+
+load_rc_config ${name}
+
+: ${vrobbler_celery_background_enable:="NO"}
+: ${vrobbler_celery_background_queues:="background"}
+: ${vrobbler_celery_background_bin:="/usr/local/bin/celery"}
+: ${vrobbler_celery_background_concurrency:="1"}
+: ${vrobbler_log_dir:="/var/log"}
+
+pidfile="/var/run/${name}.pid"
+
+command="/usr/sbin/daemon"
+command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_background_bin} -A vrobbler worker -Q ${vrobbler_celery_background_queues} -n background@%h --concurrency=${vrobbler_celery_background_concurrency} -l info --logfile ${vrobbler_log_dir}/${name}.log"
+
+run_rc_command "$1"

rc.d/vrobbler_celery_priority

@@ -0,0 +1,45 @@
+#!/bin/sh
+#
+# rc.d script for the vrobbler high-priority Celery worker.
+#
+# PROVIDE: vrobbler_celery_priority
+# REQUIRE: NETWORKING postgresql redis
+# KEYWORD: shutdown
+#
+# Installation (on the FreeBSD jail):
+#
+#   install -m 0555 rc.d/vrobbler_celery_priority /usr/local/etc/rc.d/vrobbler_celery_priority
+#   sysrc vrobbler_celery_priority_enable="YES"
+#   service vrobbler_celery_priority start
+#
+# Configuration (via sysrc or /etc/rc.conf):
+#
+#   vrobbler_celery_priority_queues  - queues to consume  (default: priority)
+#   vrobbler_celery_priority_bin     - path to celery     (default: /usr/local/bin/celery)
+#   vrobbler_celery_priority_flags   - extra daemon(8) flags
+#
+# Consumes the "priority" queue where interactive/user-facing tasks (agent
+# session prompts, Mopidy queue adds, favorites, reverse geocoding) are routed.
+# Higher concurrency with a small prefetch keeps these tasks snappy even when
+# the default and background workers are busy. The nodename is fixed to
+# priority@<hostname> so it can coexist with the other workers on one host.
+#
+. /etc/rc.subr
+
+name="vrobbler_celery_priority"
+rcvar="vrobbler_celery_priority_enable"
+
+load_rc_config ${name}
+
+: ${vrobbler_celery_priority_enable:="NO"}
+: ${vrobbler_celery_priority_queues:="priority"}
+: ${vrobbler_celery_priority_bin:="/usr/local/bin/celery"}
+: ${vrobbler_celery_priority_concurrency:="4"}
+: ${vrobbler_log_dir:="/var/log"}
+
+pidfile="/var/run/${name}.pid"
+
+command="/usr/sbin/daemon"
+command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_priority_bin} -A vrobbler worker -Q ${vrobbler_celery_priority_queues} -n priority@%h --concurrency=${vrobbler_celery_priority_concurrency} --prefetch-multiplier=1 -l info --logfile ${vrobbler_log_dir}/${name}.log"
+
+run_rc_command "$1"

vrobbler/settings.py

@@ -171,12 +171,36 @@ CELERY_RESULT_EXTENDED = True
 CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True
 
 CELERY_TASK_CREATE_MISSING_QUEUES = True
+CELERY_TASK_DEFAULT_QUEUE = "default"
+
+# Queue-based prioritization. Interactive/user-facing tasks are routed to a
+# dedicated "priority" worker so they never wait behind long-running batch
+# jobs. Heavy, scheduled off-peak work goes to a low-concurrency "background"
+# worker. Everything not listed here lands on the "default" queue.
 CELERY_TASK_ROUTES = {
+    # Interactive: user is waiting on the result.
+    "scrobbles.tasks.scrobble_agent_session_prompt": {"queue": "priority"},
+    "scrobbles.tasks.add_scrobble_to_mopidy_queue": {"queue": "priority"},
+    "scrobbles.tasks.add_scrobble_to_mopidy_monthly_playlist": {"queue": "priority"},
+    "scrobbles.tasks.add_favorite_to_mopidy_playlist": {"queue": "priority"},
+    "scrobbles.tasks.remove_favorite_from_mopidy_playlist": {"queue": "priority"},
+    "scrobbles.tasks.reverse_geocode_geolocation": {"queue": "priority"},
+    # Per-scrobble chart updates stay on the charts queue so they are not
+    # blocked behind scheduled chart rebuilds.
     "scrobbles.tasks.update_charts_for_timestamp": {"queue": "charts"},
-    "scrobbles.tasks.create_yesterdays_charts": {"queue": "charts"},
-    "scrobbles.tasks.rebuild_weekly_charts": {"queue": "charts"},
-    "scrobbles.tasks.rebuild_monthly_charts": {"queue": "charts"},
-    "scrobbles.tasks.rebuild_yearly_charts": {"queue": "charts"},
+    "scrobbles.tasks.build_charts_for_user": {"queue": "charts"},
+    # Heavy, scheduled off-peak jobs run on a dedicated low-concurrency worker.
+    "scrobbles.tasks.create_yesterdays_charts": {"queue": "background"},
+    "scrobbles.tasks.rebuild_weekly_charts": {"queue": "background"},
+    "scrobbles.tasks.rebuild_monthly_charts": {"queue": "background"},
+    "scrobbles.tasks.rebuild_yearly_charts": {"queue": "background"},
+    "scrobbles.tasks.backup_database": {"queue": "background"},
+    "scrobbles.tasks.backfill_scrobble_sentiment": {"queue": "background"},
+    "scrobbles.tasks.check_twitch_channels_for_vods": {"queue": "background"},
+    "scrobbles.importers.lastfm.dispatch_historical_imports": {"queue": "background"},
+    "trends.tasks.compute_all_trends": {"queue": "background"},
+    "trends.tasks.compute_user_trends": {"queue": "background"},
+    "trends.tasks.compute_single_trend": {"queue": "background"},
 }
 
 CELERY_BEAT_SCHEDULE = {