From 86c495f22ffd877c1430f355b0a669bddace7a9e Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Fri, 22 May 2026 11:38:22 -0400 Subject: [PATCH] [tasks] Move tasks out of cron and into celerybeat --- vrobbler/apps/scrobbles/tasks.py | 214 +++++++++++++++++++++++++++++++ vrobbler/settings.py | 33 +++++ 2 files changed, 247 insertions(+) diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index 4781e58..29924d0 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -10,6 +10,7 @@ from charts.utils import ( build_yearly_charts, ) from django.apps import apps +from django.conf import settings from django.contrib.auth import get_user_model from django.utils import timezone @@ -217,3 +218,216 @@ def update_charts_for_timestamp(user_id, year, month, day, week): logger.info(f"[charts] Updated charts for {user} on {date_str}") except Exception as e: logger.error(f"[charts] Failed to update charts: {e}") + + +# ── Crontab replacements ────────────────────────────────────────────────────── + + +BACKUP_RETENTION_DAYS = 7 +BACKUP_RETENTION_MONTHS = 12 + + +def _retention_files_to_delete(remote_files, now): + """Return list of filenames to delete under retention policy. + + Policy: keep all files from the last *BACKUP_RETENTION_DAYS* days, + plus the single most-recent backup from each of the last + *BACKUP_RETENTION_MONTHS* calendar months (not already covered by + the day-based window). + """ + import re + from collections import defaultdict + from datetime import timedelta, date + + pattern = re.compile(r"vrobbler-backup-(\d{4})_(\d{2})_(\d{2})\.sql\.gz") + parsed = [] + for fname in remote_files: + m = pattern.match(fname) + if not m: + continue + y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) + parsed.append((date(y, mo, d), fname)) + + cutoff = now.date() - timedelta(days=BACKUP_RETENTION_DAYS) + + keep = set() + for d, fname in parsed: + if d >= cutoff: + keep.add(fname) + + remaining = [(d, fname) for d, fname in parsed if fname not in keep] + by_month = defaultdict(list) + for d, fname in remaining: + by_month[(d.year, d.month)].append((d, fname)) + + for files in by_month.values(): + files.sort(key=lambda x: x[0], reverse=True) + keep.add(files[0][1]) + + return [fname for _, fname in parsed if fname not in keep] + + +def _run_remote_cleanup(ssh_key, ssh_host, remote_path): + """SSH to remote host, list backup files, delete those outside retention.""" + import subprocess + import re + + result = subprocess.run( + ["ssh", "-i", ssh_key, ssh_host, "ls", "-1", remote_path], + capture_output=True, text=True, timeout=30, + ) + if result.returncode != 0: + logger.warning( + "backup_database: could not list remote files (%s)", result.stderr.strip() + ) + return + + files = [line.strip() for line in result.stdout.splitlines() if line.strip()] + if not files: + return + + from datetime import datetime + now = datetime.now() + to_delete = _retention_files_to_delete(files, now) + if not to_delete: + logger.info("backup_database: no remote files to prune") + return + + # Delete in batches to avoid absurdly long command lines + batch_size = 50 + for i in range(0, len(to_delete), batch_size): + batch = to_delete[i : i + batch_size] + rm_cmd = ["ssh", "-i", ssh_key, ssh_host, "rm", "-f"] + rm_cmd.extend(f"{remote_path}/{f}" for f in batch) + subprocess.run(rm_cmd, check=True, timeout=30) + + logger.info( + "backup_database: pruned %d remote backup(s)", len(to_delete) + ) + + +@shared_task +def backup_database(): + """pg_dump + gzip, scp to remote, retention cleanup, ntfy notification.""" + import os + import subprocess + import re + from datetime import datetime + from pathlib import Path + + import requests as req + + db = settings.DATABASES["default"] + engine = db["ENGINE"] + if "postgresql" not in engine and "postgis" not in engine: + logger.warning("backup_database skipped — not PostgreSQL") + return + + date_str = datetime.now().strftime("%Y_%m_%d") + backup_path = Path(f"/tmp/vrobbler-backup-{date_str}.sql.gz") + + env = os.environ.copy() + if db.get("PASSWORD"): + env["PGPASSWORD"] = db["PASSWORD"] + + pg_dump_cmd = [ + "pg_dump", + "-h", db.get("HOST", "localhost"), + "-U", db.get("USER", "postgres"), + "-d", db["NAME"], + ] + + try: + with open(backup_path, "wb") as f: + dump_proc = subprocess.Popen(pg_dump_cmd, stdout=subprocess.PIPE, env=env) + gzip_proc = subprocess.Popen( + ["gzip"], stdin=dump_proc.stdout, stdout=f + ) + dump_proc.stdout.close() + gzip_proc.communicate() + + if gzip_proc.returncode != 0: + logger.error("backup_database: pg_dump / gzip failed") + return + + ssh_key = getattr(settings, "DB_BACKUP_SSH_KEY", "") + ssh_dest = getattr(settings, "DB_BACKUP_SSH_DEST", "") + if ssh_key and ssh_dest: + subprocess.run( + ["scp", "-i", ssh_key, str(backup_path), ssh_dest], + check=True, + ) + logger.info("backup_database: copied to %s", ssh_dest) + + # Parse user@host and path from dest + m = re.match(r"(\S+)@(\S+):(.+)", ssh_dest) + if m: + ssh_host = f"{m.group(1)}@{m.group(2)}" + remote_path = m.group(3) + _run_remote_cleanup(ssh_key, ssh_host, remote_path) + + ntfy_url = getattr( + settings, "DB_BACKUP_NTFY_URL", "https://ntfy.unbl.ink/backups" + ) + req.post(ntfy_url, data=b"Vrobbler backup succeeded") + logger.info("backup_database: completed %s", backup_path) + + backup_path.unlink(missing_ok=True) + except Exception as e: + logger.error("backup_database failed: %s", e) + + +@shared_task +def import_from_lastfm_all_users(): + """Import Last.fm scrobbles for all users (replaces */30 cron).""" + from scrobbles.utils import import_lastfm_for_all_users + + import_lastfm_for_all_users() + + +@shared_task +def import_from_retroarch_all_users(): + """Import RetroArch scrobbles for all users (replaces @daily cron).""" + from scrobbles.utils import import_retroarch_for_all_users + + import_retroarch_for_all_users() + + +@shared_task +def import_from_webdav_all_users(): + """Import from WebDAV for all users (replaces */2 cron).""" + from scrobbles.importers.webdav import import_from_webdav_for_all_users + + import_from_webdav_for_all_users() + + +@shared_task +def import_from_imap_all_users(): + """Import from IMAP for all users (replaces */4 cron).""" + from scrobbles.importers.imap import import_scrobbles_from_imap + + import_scrobbles_from_imap() + + +@shared_task +def import_from_lichess_all_users(): + """Import chess games from Lichess for all users (replaces */20 cron).""" + from boardgames.sources.lichess import import_chess_games_for_all_users + + import_chess_games_for_all_users() + + +@shared_task +def send_notification_for_in_progress(): + """Send ntfy stop-notifications for in-progress scrobbles (replaces */3 cron).""" + from scrobbles.utils import send_stop_notifications_for_in_progress_scrobbles + + send_stop_notifications_for_in_progress_scrobbles() + + +@shared_task +def send_mood_checkin(): + """Send mood check-in reminders (replaces @hourly cron).""" + from scrobbles.utils import send_mood_checkin_reminders + + send_mood_checkin_reminders() diff --git a/vrobbler/settings.py b/vrobbler/settings.py index 9796fd3..247a235 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -126,6 +126,39 @@ CELERY_BEAT_SCHEDULE = { "task": "scrobbles.tasks.rebuild_yearly_charts", "schedule": crontab(hour=0, minute=30, day_of_month=1, month_of_year=1), }, + # ── Crontab replacements ───────────────────────────────────────────── + "database-backup": { + "task": "scrobbles.tasks.backup_database", + "schedule": crontab(hour=0, minute=0), + }, + "import-from-lastfm": { + "task": "scrobbles.tasks.import_from_lastfm_all_users", + "schedule": crontab(minute="*/30"), + }, + "import-from-retroarch": { + "task": "scrobbles.tasks.import_from_retroarch_all_users", + "schedule": crontab(hour=0, minute=0), + }, + "import-from-webdav": { + "task": "scrobbles.tasks.import_from_webdav_all_users", + "schedule": crontab(minute="*/2"), + }, + "import-from-imap": { + "task": "scrobbles.tasks.import_from_imap_all_users", + "schedule": crontab(minute="*/4"), + }, + "import-from-lichess": { + "task": "scrobbles.tasks.import_from_lichess_all_users", + "schedule": crontab(minute="*/20"), + }, + "send-notification-for-in-progress": { + "task": "scrobbles.tasks.send_notification_for_in_progress", + "schedule": crontab(minute="*/3"), + }, + "send-mood-checkin": { + "task": "scrobbles.tasks.send_mood_checkin", + "schedule": crontab(hour="*", minute=0), + }, } INSTALLED_APPS = [