diff --git a/.drone.yml b/.drone.yml index 97bb61d..2bb816b 100644 --- a/.drone.yml +++ b/.drone.yml @@ -37,7 +37,7 @@ steps: - pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git@main - vrobbler migrate - vrobbler collectstatic --noinput - - service vrobbler_celery restart && service vrobbler_celerybeat restart && service vrobbler restart + - service vrobbler_celery restart && service vrobbler_celery_priority restart && service vrobbler_celery_background restart && service vrobbler_celerybeat restart && service vrobbler restart when: ref: - refs/tags/* diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 7e3ca9f..c5bd230 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -137,7 +137,7 @@ jobs: python3 -c "import vrobbler; print(f'vrobbler {vrobbler.__version__} installed OK')" vrobbler migrate vrobbler collectstatic --noinput - service vrobbler_celery restart && service vrobbler_celerybeat restart && service vrobbler restart + service vrobbler_celery restart && service vrobbler_celery_priority restart && service vrobbler_celery_background restart && service vrobbler_celerybeat restart && service vrobbler restart - name: Notify deploy success (ntfy) if: success() diff --git a/CHANGELOG.org b/CHANGELOG.org index df5530f..45b8a60 100644 --- a/CHANGELOG.org +++ b/CHANGELOG.org @@ -117,6 +117,34 @@ Import gym sessions and routines from a Flexify backup. cover body-composition metrics on Task scrobbles and `bodyweight_kg` on Workout scrobbles. +** DONE [#A] Fix celery task prioritization especially for agent sessions :celery:tasks:agents: +:PROPERTIES: +:ID: 672c81bf-bba9-a963-e8ac-52246d976cea +:END: + +Split Celery work into four queues so time-sensitive tasks run promptly even +when long batch jobs are queued: + +- `priority` — interactive/user-facing tasks (agent session prompts, Mopidy + queue/playlist adds, favorite toggles, reverse geocoding) get a dedicated + high-concurrency worker (`--concurrency=4 --prefetch-multiplier=1`). +- `charts` — per-scrobble chart updates (`update_charts_for_timestamp`) stay on + their own queue, no longer blocked behind scheduled chart rebuilds (those + moved to `background`). +- `background` — heavy/scheduled jobs (db backup, trend computation, sentiment + backfill, chart rebuilds, twitch VOD check, historical Last.fm dispatch) run + on a dedicated low-concurrency worker (`--concurrency=1`). +- `default` — everything else (imports, notifications, archivebox pushes, + board-game expansion fetches). `CELERY_TASK_DEFAULT_QUEUE` is now set + explicitly to `default`; previously un-routed tasks were published to + Celery's default `celery` queue, which no worker consumed. + +Workers run as three FreeBSD daemons (`vrobbler_celery` on `default,charts`, +plus new `vrobbler_celery_priority` and `vrobbler_celery_background` rc.d +scripts with distinct nodenames). Procfile, justfile, Makefile, Drone and +Gitea workflows updated accordingly. + + * Version 64.5 [1/1] ** DONE [#B] Clean up issues with org-mode notes :orgmode:notes:scrobbles: :PROPERTIES: diff --git a/Makefile b/Makefile index 47e42f9..f9c490a 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ deploy: - ssh vrobbler.service "pip uninstall vrobbler && pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git && service vrobbler restart && service vrobbler_celery restart && service vrobbler_celerybeat restart && vrobbler migrate" + ssh vrobbler.service "pip uninstall vrobbler && pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git && service vrobbler restart && service vrobbler_celery restart && service vrobbler_celery_priority restart && service vrobbler_celery_background restart && service vrobbler_celerybeat restart && vrobbler migrate" logs: ssh life.unbl.ink tail -n 100 -f /var/log/vrobbler.json test: diff --git a/PROJECT.org b/PROJECT.org index 07405e4..e49b1af 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -18,7 +18,7 @@ tasks, Todoist tasks, web pages I've read and trails I've hiked has turned out to be sometimes cathartic and sometimes functional as I try to remember when I did a thing. -* Backlog [0/24] :vrobbler:project:personal: +* Backlog [2/26] :vrobbler:project:personal: ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata: :PROPERTIES: :ID: d3cce1a7-d540-4d66-bf66-e75378e4eac7 @@ -556,4 +556,32 @@ Added a `workouts` Django app so we can scrobble gym sessions. - Templates, admin, DRF viewsets (`exercises`, `workout-routines`), MCP tools, and `tests/workouts_tests/` (unit conversion, form round-trips, importer, imperial POST) included. +** DONE [#A] Fix celery task prioritization especially for agent sessions :celery:tasks:agents: +:PROPERTIES: +:ID: 672c81bf-bba9-a963-e8ac-52246d976cea +:END: +*** Description + +Split Celery work into four queues so time-sensitive tasks run fast even when +long batch jobs are queued up: + +- `priority` — interactive/user-facing tasks (agent session prompts, Mopidy + queue/playlist adds, favorite toggles, reverse geocoding) get a dedicated + high-concurrency worker with `--prefetch-multiplier=1`. +- `charts` — per-scrobble chart updates stay on their own queue, no longer + blocked by scheduled chart rebuilds. +- `background` — heavy/scheduled jobs (db backup, trends, sentiment backfill, + chart rebuilds, historical imports, twitch VOD check) run on a dedicated + low-concurrency worker so they can't starve the others. +- `default` — everything else (imports, notifications, archivebox, expansions). + +Also set `CELERY_TASK_DEFAULT_QUEUE = "default"` — previously un-routed tasks +were published to Celery's default `celery` queue, which no worker consumed. +New rc.d scripts `vrobbler_celery_priority` and `vrobbler_celery_background` +added alongside the existing `vrobbler_celery` (now `default,charts`). + +** DONE [#A] Fix lastfm rate limiting dropping scrobbles :importers:scrobbles:lastfm: +:PROPERTIES: +:ID: e6718364-8f9e-d156-3a79-2e89b2ac697b +:END: diff --git a/Procfile b/Procfile index ce899cf..a28a3ba 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,4 @@ web: python manage.py runserver 0.0.0.0:8014 worker: celery -A vrobbler worker -Q default,charts -l DEBUG +worker-priority: celery -A vrobbler worker -Q priority --concurrency=4 --prefetch-multiplier=1 -l DEBUG +worker-background: celery -A vrobbler worker -Q background --concurrency=1 -l DEBUG diff --git a/justfile b/justfile index b323e69..b38ecf3 100644 --- a/justfile +++ b/justfile @@ -10,7 +10,13 @@ shell: poetry run python manage.py shell celery: - poetry run celery -A vrobbler worker -l info --concurrency=2 --pool=threads + poetry run celery -A vrobbler worker -Q default,charts -l info --concurrency=2 --pool=threads + +celery-priority: + poetry run celery -A vrobbler worker -Q priority -l info --concurrency=2 --pool=threads + +celery-background: + poetry run celery -A vrobbler worker -Q background -l info --concurrency=1 --pool=threads celery-beat: poetry run celery -A vrobbler beat -l info diff --git a/rc.d/vrobbler_celery b/rc.d/vrobbler_celery index 90ad283..c20239f 100755 --- a/rc.d/vrobbler_celery +++ b/rc.d/vrobbler_celery @@ -29,9 +29,10 @@ # is what `service vrobbler_celery stop` does) forwards the signal to celery, # which shuts the worker down gracefully. # -# The queues match the old immortal/Procfile setup. If you ever run more than -# one worker instance, give each a unique nodename via -n (celery@%h is the -# hostname default). +# The default,charts worker handles on-demand tasks plus per-scrobble chart +# updates. Companion workers (vrobbler_celery_priority, vrobbler_celery_background) +# consume the priority and background queues with distinct nodenames so they can +# run side by side on one host. # . /etc/rc.subr @@ -49,6 +50,6 @@ load_rc_config ${name} pidfile="/var/run/${name}.pid" command="/usr/sbin/daemon" -command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_bin} -A vrobbler worker -Q ${vrobbler_celery_queues} -n celery@%h -l info --logfile ${vrobbler_log_dir}/${name}.log" +command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_bin} -A vrobbler worker -Q ${vrobbler_celery_queues} -n default@%h -l info --logfile ${vrobbler_log_dir}/${name}.log" run_rc_command "$1" diff --git a/rc.d/vrobbler_celery_background b/rc.d/vrobbler_celery_background new file mode 100755 index 0000000..fb82331 --- /dev/null +++ b/rc.d/vrobbler_celery_background @@ -0,0 +1,45 @@ +#!/bin/sh +# +# rc.d script for the vrobbler background Celery worker. +# +# PROVIDE: vrobbler_celery_background +# REQUIRE: NETWORKING postgresql redis +# KEYWORD: shutdown +# +# Installation (on the FreeBSD jail): +# +# install -m 0555 rc.d/vrobbler_celery_background /usr/local/etc/rc.d/vrobbler_celery_background +# sysrc vrobbler_celery_background_enable="YES" +# service vrobbler_celery_background start +# +# Configuration (via sysrc or /etc/rc.conf): +# +# vrobbler_celery_background_queues - queues to consume (default: background) +# vrobbler_celery_background_bin - path to celery (default: /usr/local/bin/celery) +# vrobbler_celery_background_flags - extra daemon(8) flags +# +# Consumes the "background" queue where heavy, scheduled off-peak jobs are +# routed (database backups, trend computation, sentiment backfill, chart +# rebuilds, historical imports). Low concurrency keeps these CPU/I/O heavy jobs +# from starving the other workers. The nodename is fixed to +# background@ so it can coexist with the other workers on one host. +# +. /etc/rc.subr + +name="vrobbler_celery_background" +rcvar="vrobbler_celery_background_enable" + +load_rc_config ${name} + +: ${vrobbler_celery_background_enable:="NO"} +: ${vrobbler_celery_background_queues:="background"} +: ${vrobbler_celery_background_bin:="/usr/local/bin/celery"} +: ${vrobbler_celery_background_concurrency:="1"} +: ${vrobbler_log_dir:="/var/log"} + +pidfile="/var/run/${name}.pid" + +command="/usr/sbin/daemon" +command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_background_bin} -A vrobbler worker -Q ${vrobbler_celery_background_queues} -n background@%h --concurrency=${vrobbler_celery_background_concurrency} -l info --logfile ${vrobbler_log_dir}/${name}.log" + +run_rc_command "$1" diff --git a/rc.d/vrobbler_celery_priority b/rc.d/vrobbler_celery_priority new file mode 100755 index 0000000..54c42d2 --- /dev/null +++ b/rc.d/vrobbler_celery_priority @@ -0,0 +1,45 @@ +#!/bin/sh +# +# rc.d script for the vrobbler high-priority Celery worker. +# +# PROVIDE: vrobbler_celery_priority +# REQUIRE: NETWORKING postgresql redis +# KEYWORD: shutdown +# +# Installation (on the FreeBSD jail): +# +# install -m 0555 rc.d/vrobbler_celery_priority /usr/local/etc/rc.d/vrobbler_celery_priority +# sysrc vrobbler_celery_priority_enable="YES" +# service vrobbler_celery_priority start +# +# Configuration (via sysrc or /etc/rc.conf): +# +# vrobbler_celery_priority_queues - queues to consume (default: priority) +# vrobbler_celery_priority_bin - path to celery (default: /usr/local/bin/celery) +# vrobbler_celery_priority_flags - extra daemon(8) flags +# +# Consumes the "priority" queue where interactive/user-facing tasks (agent +# session prompts, Mopidy queue adds, favorites, reverse geocoding) are routed. +# Higher concurrency with a small prefetch keeps these tasks snappy even when +# the default and background workers are busy. The nodename is fixed to +# priority@ so it can coexist with the other workers on one host. +# +. /etc/rc.subr + +name="vrobbler_celery_priority" +rcvar="vrobbler_celery_priority_enable" + +load_rc_config ${name} + +: ${vrobbler_celery_priority_enable:="NO"} +: ${vrobbler_celery_priority_queues:="priority"} +: ${vrobbler_celery_priority_bin:="/usr/local/bin/celery"} +: ${vrobbler_celery_priority_concurrency:="4"} +: ${vrobbler_log_dir:="/var/log"} + +pidfile="/var/run/${name}.pid" + +command="/usr/sbin/daemon" +command_args="-f -P ${pidfile} -R 5 -t ${name} ${vrobbler_celery_priority_bin} -A vrobbler worker -Q ${vrobbler_celery_priority_queues} -n priority@%h --concurrency=${vrobbler_celery_priority_concurrency} --prefetch-multiplier=1 -l info --logfile ${vrobbler_log_dir}/${name}.log" + +run_rc_command "$1" diff --git a/vrobbler/settings.py b/vrobbler/settings.py index e62d8ec..2101530 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -171,12 +171,36 @@ CELERY_RESULT_EXTENDED = True CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True CELERY_TASK_CREATE_MISSING_QUEUES = True +CELERY_TASK_DEFAULT_QUEUE = "default" + +# Queue-based prioritization. Interactive/user-facing tasks are routed to a +# dedicated "priority" worker so they never wait behind long-running batch +# jobs. Heavy, scheduled off-peak work goes to a low-concurrency "background" +# worker. Everything not listed here lands on the "default" queue. CELERY_TASK_ROUTES = { + # Interactive: user is waiting on the result. + "scrobbles.tasks.scrobble_agent_session_prompt": {"queue": "priority"}, + "scrobbles.tasks.add_scrobble_to_mopidy_queue": {"queue": "priority"}, + "scrobbles.tasks.add_scrobble_to_mopidy_monthly_playlist": {"queue": "priority"}, + "scrobbles.tasks.add_favorite_to_mopidy_playlist": {"queue": "priority"}, + "scrobbles.tasks.remove_favorite_from_mopidy_playlist": {"queue": "priority"}, + "scrobbles.tasks.reverse_geocode_geolocation": {"queue": "priority"}, + # Per-scrobble chart updates stay on the charts queue so they are not + # blocked behind scheduled chart rebuilds. "scrobbles.tasks.update_charts_for_timestamp": {"queue": "charts"}, - "scrobbles.tasks.create_yesterdays_charts": {"queue": "charts"}, - "scrobbles.tasks.rebuild_weekly_charts": {"queue": "charts"}, - "scrobbles.tasks.rebuild_monthly_charts": {"queue": "charts"}, - "scrobbles.tasks.rebuild_yearly_charts": {"queue": "charts"}, + "scrobbles.tasks.build_charts_for_user": {"queue": "charts"}, + # Heavy, scheduled off-peak jobs run on a dedicated low-concurrency worker. + "scrobbles.tasks.create_yesterdays_charts": {"queue": "background"}, + "scrobbles.tasks.rebuild_weekly_charts": {"queue": "background"}, + "scrobbles.tasks.rebuild_monthly_charts": {"queue": "background"}, + "scrobbles.tasks.rebuild_yearly_charts": {"queue": "background"}, + "scrobbles.tasks.backup_database": {"queue": "background"}, + "scrobbles.tasks.backfill_scrobble_sentiment": {"queue": "background"}, + "scrobbles.tasks.check_twitch_channels_for_vods": {"queue": "background"}, + "scrobbles.importers.lastfm.dispatch_historical_imports": {"queue": "background"}, + "trends.tasks.compute_all_trends": {"queue": "background"}, + "trends.tasks.compute_user_trends": {"queue": "background"}, + "trends.tasks.compute_single_trend": {"queue": "background"}, } CELERY_BEAT_SCHEDULE = {