import logging from datetime import timedelta from celery import shared_task from charts.utils import ( build_charts_since, build_daily_charts, build_monthly_charts, build_weekly_charts, build_yearly_charts, ) from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model from django.utils import timezone logger = logging.getLogger(__name__) User = get_user_model() CHARTABLE_MEDIA_TYPES = [ "artist", "album", "track", "tv_series", "video", "podcast", "board_game", "trail", "food", "book", ] SCROBBLES_WITHOUT_CHARTS = [ "geolocation", ] @shared_task def check_twitch_channels_for_vods(): """Check recent Twitch channel scrobbles for matching VODs.""" from scrobbles.models import Scrobble cutoff = timezone.now() - timedelta(hours=48) channel_scrobbles = Scrobble.objects.filter( media_type=Scrobble.MediaType.CHANNEL, timestamp__gte=cutoff, ) logger.info(f"[twitch_vods] Checking {channel_scrobbles.count()} channel scrobbles") matched_count = 0 for scrobble in channel_scrobbles: if not scrobble.channel or not scrobble.channel.twitch_id: continue try: from videos.sources.twitch import get_channel_vods vods = get_channel_vods(scrobble.channel.twitch_id) scrobble_time = scrobble.timestamp for vod in vods: if not vod.get("published_at"): continue from videos.sources import twitch as twitch_source vod_time = twitch_source.parse_twitch_datetime(vod["published_at"]) if not vod_time: continue time_diff = (vod_time - scrobble_time).total_seconds() if 0 < time_diff <= 86400: from videos.models import Video video = Video.get_from_twitch_id(vod["id"], overwrite=True) if video: video.scrobble_for_user( scrobble.user_id, status="stopped", source="Twitch VOD", ) matched_count += 1 logger.info( f"[twitch_vods] Matched VOD {vod['id']} for channel scrobble {scrobble.id}" ) break except Exception as e: logger.warning( f"[twitch_vods] Error processing scrobble {scrobble.id}: {e}" ) logger.info(f"[twitch_vods] Matched {matched_count} VODs") @shared_task def process_retroarch_import(import_id): RetroarchImport = apps.get_model("scrobbles", "RetroarchImport") retroarch_import = RetroarchImport.objects.filter(id=import_id).first() if not retroarch_import: logger.warn(f"RetroarchImport not found with id {import_id}") retroarch_import.process() @shared_task def process_bgstats_import(import_id): BGStatsImport = apps.get_model("scrobbles", "BGStatsImport") bgstats_import = BGStatsImport.objects.filter(id=import_id).first() if not bgstats_import: logger.warn(f"BGStatsImport not found with id {import_id}") return bgstats_import.process() @shared_task def process_lastfm_import(import_id): LastFmImport = apps.get_model("scrobbles", "LastFMImport") lastfm_import = LastFmImport.objects.filter(id=import_id).first() if not lastfm_import: logger.warn(f"LastFmImport not found with id {import_id}") lastfm_import.process() @shared_task def process_tsv_import(import_id): model_path = "scrobbles.AudioscrobblerTSVImport" AudioScrobblerTSVImport = apps.get_model(model_path) tsv_import = AudioScrobblerTSVImport.objects.filter(id=import_id).first() if not tsv_import: logger.warn(f"AudioScrobblerTSVImport not found with id {import_id}") tsv_import.process() @shared_task def process_koreader_import(import_id): KoReaderImport = apps.get_model("scrobbles", "KoReaderImport") koreader_import = KoReaderImport.objects.filter(id=import_id).first() if not koreader_import: logger.warn(f"KOReaderImport not found with id {import_id}") koreader_import.process() @shared_task def process_trail_gpx_import(import_id): TrailGPXImport = apps.get_model("scrobbles", "TrailGPXImport") trail_gpx_import = TrailGPXImport.objects.filter(id=import_id).first() if not trail_gpx_import: logger.warn(f"TrailGPXImport not found with id {import_id}") return trail_gpx_import.process() @shared_task def process_ebird_csv_import(import_id): EBirdCSVImport = apps.get_model("scrobbles", "EBirdCSVImport") birding_import = EBirdCSVImport.objects.filter(id=import_id).first() if not birding_import: logger.warn(f"EBirdCSVImport not found with id {import_id}") return birding_import.process() @shared_task def process_scale_csv_import(import_id): ScaleCSVImport = apps.get_model("scrobbles", "ScaleCSVImport") scale_import = ScaleCSVImport.objects.filter(id=import_id).first() if not scale_import: logger.warn(f"ScaleCSVImport not found with id {import_id}") return scale_import.process() @shared_task def create_yesterdays_charts(): """Build/update charts for all users starting from last known record.""" for user in User.objects.all(): build_charts_since(user) @shared_task def build_charts_for_user(user_id): """Build charts for a specific user starting from last known record.""" user = User.objects.filter(id=user_id).first() if not user: logger.error(f"User with id {user_id} not found") return logger.info(f"Building charts for {user}") build_charts_since(user) @shared_task def rebuild_weekly_charts(): """Rebuild weekly charts for all users for the just-completed week.""" now = timezone.now() year, week, _ = now.isocalendar() for user in User.objects.all(): build_weekly_charts(user, year, week, CHARTABLE_MEDIA_TYPES) logger.info(f"Rebuilt weekly charts for week {week}, {year}") @shared_task def rebuild_monthly_charts(): """Rebuild monthly charts for all users for the just-completed month.""" now = timezone.now() year, month = now.year, now.month if month == 1: month = 12 year -= 1 else: month -= 1 for user in User.objects.all(): build_monthly_charts(user, year, month, CHARTABLE_MEDIA_TYPES) logger.info(f"Rebuilt monthly charts for {month}/{year}") @shared_task def rebuild_yearly_charts(): """Rebuild yearly charts for all users for the just-completed year.""" now = timezone.now() year = now.year - 1 for user in User.objects.all(): build_yearly_charts(user, year, CHARTABLE_MEDIA_TYPES) logger.info(f"Rebuilt yearly charts for {year}") @shared_task def update_charts_for_timestamp(user_id, year, month, day, week): """Update charts for a specific time period.""" user = User.objects.filter(id=user_id).first() if not user: logger.error(f"User with id {user_id} not found") return try: build_daily_charts(user, year, month, day, CHARTABLE_MEDIA_TYPES) build_weekly_charts(user, year, week, CHARTABLE_MEDIA_TYPES) build_monthly_charts(user, year, month, CHARTABLE_MEDIA_TYPES) build_yearly_charts(user, year, CHARTABLE_MEDIA_TYPES) date_str = f"{year}-{month:02d}-{day:02d}" logger.info(f"[charts] Updated charts for {user} on {date_str}") except Exception as e: logger.error(f"[charts] Failed to update charts: {e}") # ── Crontab replacements ────────────────────────────────────────────────────── BACKUP_RETENTION_DAYS = 7 BACKUP_RETENTION_MONTHS = 12 def _cleanup_failed_backup(backup_path): """Remove a failed/incomplete backup file if it exists.""" from pathlib import Path p = Path(backup_path) if p.exists(): p.unlink() logger.warning("backup_database: removed incomplete backup %s", backup_path) def _retention_files_to_delete(remote_files, now): """Return list of filenames to delete under retention policy. Policy: keep all files from the last *BACKUP_RETENTION_DAYS* days, plus the single most-recent backup from each of the last *BACKUP_RETENTION_MONTHS* calendar months (not already covered by the day-based window). """ import re from collections import defaultdict from datetime import timedelta, date pattern = re.compile(r"vrobbler-backup-(\d{4})_(\d{2})_(\d{2})\.sql\.gz") parsed = [] for fname in remote_files: m = pattern.match(fname) if not m: continue y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3)) parsed.append((date(y, mo, d), fname)) cutoff = now.date() - timedelta(days=BACKUP_RETENTION_DAYS) keep = set() for d, fname in parsed: if d >= cutoff: keep.add(fname) remaining = [(d, fname) for d, fname in parsed if fname not in keep] by_month = defaultdict(list) for d, fname in remaining: by_month[(d.year, d.month)].append((d, fname)) for files in by_month.values(): files.sort(key=lambda x: x[0], reverse=True) keep.add(files[0][1]) return [fname for _, fname in parsed if fname not in keep] def _run_remote_cleanup(ssh_key, ssh_host, remote_path): """SSH to remote host, list backup files, delete those outside retention.""" import subprocess import re result = subprocess.run( ["ssh", "-i", ssh_key, ssh_host, "ls", "-1", remote_path], capture_output=True, text=True, timeout=30, ) if result.returncode != 0: logger.warning( "backup_database: could not list remote files (%s)", result.stderr.strip() ) return files = [line.strip() for line in result.stdout.splitlines() if line.strip()] if not files: return from datetime import datetime now = datetime.now() to_delete = _retention_files_to_delete(files, now) if not to_delete: logger.info("backup_database: no remote files to prune") return # Delete in batches to avoid absurdly long command lines batch_size = 50 for i in range(0, len(to_delete), batch_size): batch = to_delete[i : i + batch_size] rm_cmd = ["ssh", "-i", ssh_key, ssh_host, "rm", "-f"] rm_cmd.extend(f"{remote_path}/{f}" for f in batch) subprocess.run(rm_cmd, check=True, timeout=30) logger.info( "backup_database: pruned %d remote backup(s)", len(to_delete) ) @shared_task def backup_database(): """pg_dump + gzip, scp to remote, retention cleanup, ntfy notification.""" import os import subprocess import re from datetime import datetime from pathlib import Path import requests as req db = settings.DATABASES["default"] engine = db["ENGINE"] if "postgresql" not in engine and "postgis" not in engine: logger.warning("backup_database skipped — not PostgreSQL") return backup_dir = Path(settings.DB_BACKUP_LOCAL_DIR) backup_dir.mkdir(parents=True, exist_ok=True) date_str = datetime.now().strftime("%Y_%m_%d") backup_path = backup_dir / f"vrobbler-backup-{date_str}.sql.gz" env = os.environ.copy() if db.get("PASSWORD"): env["PGPASSWORD"] = db["PASSWORD"] pg_dump_cmd = [ "pg_dump", "--no-blobs", "-h", db.get("HOST", "localhost"), "-U", db.get("USER", "postgres"), "-d", db["NAME"], ] logger.info("backup_database: dumping %s to %s", db["NAME"], backup_path) try: with open(backup_path, "wb") as f: dump_proc = subprocess.Popen(pg_dump_cmd, stdout=subprocess.PIPE, env=env) gzip_proc = subprocess.Popen( ["gzip"], stdin=dump_proc.stdout, stdout=f ) dump_proc.stdout.close() gzip_proc.communicate() if gzip_proc.returncode != 0: logger.error("backup_database: pg_dump / gzip failed") _cleanup_failed_backup(backup_path) return dump_size = backup_path.stat().st_size logger.info( "backup_database: dump complete (%.1f MB)", dump_size / 1_000_000 ) ssh_key = getattr(settings, "DB_BACKUP_SSH_KEY", "") ssh_dest = getattr(settings, "DB_BACKUP_SSH_DEST", "") if ssh_key and ssh_dest: logger.info("backup_database: copying to %s", ssh_dest) try: subprocess.run( ["scp", "-i", ssh_key, str(backup_path), ssh_dest], check=True, capture_output=True, text=True, ) except subprocess.CalledProcessError as exc: logger.error( "backup_database: scp failed (stderr: %s)", exc.stderr.strip() ) _cleanup_failed_backup(backup_path) return logger.info("backup_database: copied to %s", ssh_dest) # Parse user@host and path from dest m = re.match(r"(\S+)@(\S+):(.+)", ssh_dest) if m: ssh_host = f"{m.group(1)}@{m.group(2)}" remote_path = m.group(3) logger.info("backup_database: pruning old remote backups") _run_remote_cleanup(ssh_key, ssh_host, remote_path) else: logger.warning( "backup_database: DB_BACKUP_SSH_KEY and DB_BACKUP_SSH_DEST not set — " "backup saved locally at %s", backup_path, ) ntfy_url = getattr( settings, "DB_BACKUP_NTFY_URL", "https://ntfy.unbl.ink/backups" ) req.post(ntfy_url, data=b"Vrobbler backup succeeded") logger.info("backup_database: completed %s", backup_path) except Exception as e: logger.error("backup_database failed: %s", e) _cleanup_failed_backup(backup_path) @shared_task def import_from_lastfm_all_users(): """Import Last.fm scrobbles for all users (replaces */30 cron).""" from vrobbler.apps.scrobbles.utils import import_lastfm_for_all_users import_lastfm_for_all_users() @shared_task def import_from_retroarch_all_users(): """Import RetroArch scrobbles for all users (replaces @daily cron). Deprecated: retroarch .lrtl files are now picked up by the WebDAV importer (scan_webdav_for_retroarch). This task remains for manual use. """ import warnings warnings.warn( "import_from_retroarch_all_users is deprecated. " "Upload .lrtl files to WebDAV var/retroarch/ instead.", DeprecationWarning, stacklevel=2, ) from vrobbler.apps.scrobbles.utils import import_retroarch_for_all_users import_retroarch_for_all_users() @shared_task def import_from_webdav_all_users(): """Import from WebDAV for all users (replaces */2 cron).""" from vrobbler.apps.scrobbles.importers.webdav import import_from_webdav_for_all_users import_from_webdav_for_all_users() @shared_task def import_from_imap_all_users(): """Deprecated — BG Stats files now picked up from WebDAV var/bgstats/.""" import warnings warnings.warn( "IMAP import is deprecated. Upload .bgsplay files to WebDAV var/bgstats/ instead.", DeprecationWarning, stacklevel=2, ) logger.warning("Skipping deprecated IMAP import (use WebDAV var/bgstats/ instead)") @shared_task def import_from_lichess_all_users(): """Import chess games from Lichess for all users (replaces */20 cron).""" from vrobbler.apps.boardgames.sources.lichess import import_chess_games_for_all_users import_chess_games_for_all_users() @shared_task def send_notification_for_in_progress(): """Send ntfy stop-notifications for in-progress scrobbles (replaces */3 cron).""" from vrobbler.apps.scrobbles.utils import send_stop_notifications_for_in_progress_scrobbles send_stop_notifications_for_in_progress_scrobbles() @shared_task def send_mood_checkin(): """Send mood check-in reminders (replaces @hourly cron).""" from vrobbler.apps.scrobbles.utils import send_mood_checkin_reminders send_mood_checkin_reminders()