Files
vrobbler/vrobbler/apps/scrobbles/tasks.py
Colin Powell 033239260f
All checks were successful
build / test (push) Successful in 1m57s
[perf] Add caching and lock protections
2026-06-19 13:37:30 -04:00

719 lines
22 KiB
Python

import logging
from datetime import datetime, timedelta
from celery import shared_task
from charts.utils import (
build_charts_since,
build_daily_charts,
build_monthly_charts,
build_weekly_charts,
build_yearly_charts,
)
from django.apps import apps
from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.cache import cache
from django.db import models
from django.utils import timezone
logger = logging.getLogger(__name__)
User = get_user_model()
CHARTABLE_MEDIA_TYPES = [
"artist",
"album",
"track",
"tv_series",
"video",
"podcast",
"board_game",
"trail",
"food",
"book",
]
SCROBBLES_WITHOUT_CHARTS = [
"geolocation",
]
@shared_task
def check_twitch_channels_for_vods():
"""Check recent Twitch channel scrobbles for matching VODs."""
from scrobbles.models import Scrobble
cutoff = timezone.now() - timedelta(hours=48)
channel_scrobbles = Scrobble.objects.filter(
media_type=Scrobble.MediaType.CHANNEL,
timestamp__gte=cutoff,
)
logger.info(f"[twitch_vods] Checking {channel_scrobbles.count()} channel scrobbles")
matched_count = 0
for scrobble in channel_scrobbles:
if not scrobble.channel or not scrobble.channel.twitch_id:
continue
try:
from videos.sources.twitch import get_channel_vods
vods = get_channel_vods(scrobble.channel.twitch_id)
scrobble_time = scrobble.timestamp
for vod in vods:
if not vod.get("published_at"):
continue
from videos.sources import twitch as twitch_source
vod_time = twitch_source.parse_twitch_datetime(vod["published_at"])
if not vod_time:
continue
time_diff = (vod_time - scrobble_time).total_seconds()
if 0 < time_diff <= 86400:
from videos.models import Video
video = Video.get_from_twitch_id(vod["id"], overwrite=True)
if video:
video.scrobble_for_user(
scrobble.user_id,
status="stopped",
source="Twitch VOD",
)
matched_count += 1
logger.info(
f"[twitch_vods] Matched VOD {vod['id']} for channel scrobble {scrobble.id}"
)
break
except Exception as e:
logger.warning(
f"[twitch_vods] Error processing scrobble {scrobble.id}: {e}"
)
logger.info(f"[twitch_vods] Matched {matched_count} VODs")
@shared_task
def process_retroarch_import(import_id):
RetroarchImport = apps.get_model("scrobbles", "RetroarchImport")
retroarch_import = RetroarchImport.objects.filter(id=import_id).first()
if not retroarch_import:
logger.warn(f"RetroarchImport not found with id {import_id}")
retroarch_import.process()
@shared_task
def process_bgstats_import(import_id):
BGStatsImport = apps.get_model("scrobbles", "BGStatsImport")
bgstats_import = BGStatsImport.objects.filter(id=import_id).first()
if not bgstats_import:
logger.warn(f"BGStatsImport not found with id {import_id}")
return
bgstats_import.process()
@shared_task
def process_lastfm_import(import_id, time_from=None, time_to=None):
LastFmImport = apps.get_model("scrobbles", "LastFMImport")
lastfm_import = LastFmImport.objects.filter(id=import_id).first()
if not lastfm_import:
logger.warn(f"LastFmImport not found with id {import_id}")
return
lastfm_import.process(time_from=time_from, time_to=time_to)
@shared_task
def process_tsv_import(import_id):
model_path = "scrobbles.AudioscrobblerTSVImport"
AudioScrobblerTSVImport = apps.get_model(model_path)
tsv_import = AudioScrobblerTSVImport.objects.filter(id=import_id).first()
if not tsv_import:
logger.warn(f"AudioScrobblerTSVImport not found with id {import_id}")
tsv_import.process()
@shared_task
def process_koreader_import(import_id):
KoReaderImport = apps.get_model("scrobbles", "KoReaderImport")
koreader_import = KoReaderImport.objects.filter(id=import_id).first()
if not koreader_import:
logger.warn(f"KOReaderImport not found with id {import_id}")
koreader_import.process()
@shared_task
def process_trail_gpx_import(import_id):
TrailGPXImport = apps.get_model("scrobbles", "TrailGPXImport")
trail_gpx_import = TrailGPXImport.objects.filter(id=import_id).first()
if not trail_gpx_import:
logger.warn(f"TrailGPXImport not found with id {import_id}")
return
trail_gpx_import.process()
@shared_task
def process_ebird_csv_import(import_id):
EBirdCSVImport = apps.get_model("scrobbles", "EBirdCSVImport")
birding_import = EBirdCSVImport.objects.filter(id=import_id).first()
if not birding_import:
logger.warn(f"EBirdCSVImport not found with id {import_id}")
return
birding_import.process()
@shared_task
def process_scale_csv_import(import_id):
ScaleCSVImport = apps.get_model("scrobbles", "ScaleCSVImport")
scale_import = ScaleCSVImport.objects.filter(id=import_id).first()
if not scale_import:
logger.warn(f"ScaleCSVImport not found with id {import_id}")
return
scale_import.process()
@shared_task
def create_yesterdays_charts():
"""Build/update charts for all users starting from last known record."""
for user in User.objects.all():
build_charts_since(user)
@shared_task
def build_charts_for_user(user_id):
"""Build charts for a specific user starting from last known record."""
user = User.objects.filter(id=user_id).first()
if not user:
logger.error(f"User with id {user_id} not found")
return
logger.info(f"Building charts for {user}")
build_charts_since(user)
@shared_task
def rebuild_weekly_charts():
"""Rebuild weekly charts for all users for the just-completed week."""
now = timezone.now()
year, week, _ = now.isocalendar()
for user in User.objects.all():
build_weekly_charts(user, year, week, CHARTABLE_MEDIA_TYPES)
logger.info(f"Rebuilt weekly charts for week {week}, {year}")
@shared_task
def rebuild_monthly_charts():
"""Rebuild monthly charts for all users for the just-completed month."""
now = timezone.now()
year, month = now.year, now.month
if month == 1:
month = 12
year -= 1
else:
month -= 1
for user in User.objects.all():
build_monthly_charts(user, year, month, CHARTABLE_MEDIA_TYPES)
logger.info(f"Rebuilt monthly charts for {month}/{year}")
@shared_task
def rebuild_yearly_charts():
"""Rebuild yearly charts for all users for the just-completed year."""
now = timezone.now()
year = now.year - 1
for user in User.objects.all():
build_yearly_charts(user, year, CHARTABLE_MEDIA_TYPES)
logger.info(f"Rebuilt yearly charts for {year}")
@shared_task
def update_charts_for_timestamp(user_id, year, month, day, week):
"""Update charts for a specific time period."""
user = User.objects.filter(id=user_id).first()
if not user:
logger.error(f"User with id {user_id} not found")
return
lock_key = f"chart_update_running_{user_id}"
if not cache.add(lock_key, "locked", timeout=300):
logger.info(f"Chart update already running for user {user_id}, skipping")
return
try:
build_daily_charts(user, year, month, day, CHARTABLE_MEDIA_TYPES)
build_weekly_charts(user, year, week, CHARTABLE_MEDIA_TYPES)
build_monthly_charts(user, year, month, CHARTABLE_MEDIA_TYPES)
build_yearly_charts(user, year, CHARTABLE_MEDIA_TYPES)
date_str = f"{year}-{month:02d}-{day:02d}"
logger.info(f"[charts] Updated charts for {user} on {date_str}")
except Exception as e:
logger.error(f"[charts] Failed to update charts: {e}")
finally:
cache.delete(lock_key)
@shared_task
def push_scrobble_to_archivebox(scrobble_id):
from scrobbles.models import Scrobble
scrobble = Scrobble.objects.filter(id=scrobble_id).first()
if not scrobble:
logger.warning(
"Scrobble %s not found for archivebox push", scrobble_id
)
return
webpage = scrobble.web_page
if not webpage:
logger.warning(
"Scrobble %s has no web_page for archivebox push", scrobble_id
)
return
webpage.push_to_archivebox(scrobble.user)
# ── Crontab replacements ──────────────────────────────────────────────────────
BACKUP_RETENTION_DAYS = 7
BACKUP_RETENTION_MONTHS = 12
def _cleanup_failed_backup(backup_path):
"""Remove a failed/incomplete backup file if it exists."""
from pathlib import Path
p = Path(backup_path)
if p.exists():
p.unlink()
logger.warning("backup_database: removed incomplete backup %s", backup_path)
def _retention_files_to_delete(remote_files, now):
"""Return list of filenames to delete under retention policy.
Policy: keep all files from the last *BACKUP_RETENTION_DAYS* days,
plus the single most-recent backup from each of the last
*BACKUP_RETENTION_MONTHS* calendar months (not already covered by
the day-based window).
"""
import re
from collections import defaultdict
from datetime import date, timedelta
pattern = re.compile(r"vrobbler-backup-(\d{4})_(\d{2})_(\d{2})\.sql\.gz")
parsed = []
for fname in remote_files:
m = pattern.match(fname)
if not m:
continue
y, mo, d = int(m.group(1)), int(m.group(2)), int(m.group(3))
parsed.append((date(y, mo, d), fname))
cutoff = now.date() - timedelta(days=BACKUP_RETENTION_DAYS)
keep = set()
for d, fname in parsed:
if d >= cutoff:
keep.add(fname)
remaining = [(d, fname) for d, fname in parsed if fname not in keep]
by_month = defaultdict(list)
for d, fname in remaining:
by_month[(d.year, d.month)].append((d, fname))
for files in by_month.values():
files.sort(key=lambda x: x[0], reverse=True)
keep.add(files[0][1])
return [fname for _, fname in parsed if fname not in keep]
def _run_remote_cleanup(ssh_key, ssh_host, remote_path):
"""SSH to remote host, list backup files, delete those outside retention.
Returns a summary string (e.g. "pruned 3 old backup(s)") or None.
"""
import re
import subprocess
result = subprocess.run(
["ssh", "-i", ssh_key, ssh_host, "ls", "-1", remote_path],
capture_output=True,
text=True,
timeout=30,
)
if result.returncode != 0:
logger.warning(
"backup_database: could not list remote files (%s)", result.stderr.strip()
)
return None
files = [line.strip() for line in result.stdout.splitlines() if line.strip()]
if not files:
return None
from datetime import datetime
now = datetime.now()
to_delete = _retention_files_to_delete(files, now)
if not to_delete:
logger.info("backup_database: no remote files to prune")
return None
# Delete in batches to avoid absurdly long command lines
batch_size = 50
for i in range(0, len(to_delete), batch_size):
batch = to_delete[i : i + batch_size]
rm_cmd = ["ssh", "-i", ssh_key, ssh_host, "rm", "-f"]
rm_cmd.extend(f"{remote_path}/{f}" for f in batch)
subprocess.run(rm_cmd, check=True, timeout=30)
logger.info("backup_database: pruned %d remote backup(s)", len(to_delete))
return f"pruned {len(to_delete)} old remote backup(s)"
@shared_task
def backup_database():
"""pg_dump + gzip, scp to remote, retention cleanup, ntfy notification."""
import os
import re
import subprocess
from datetime import datetime
from pathlib import Path
import requests as req
db = settings.DATABASES["default"]
engine = db["ENGINE"]
if "postgresql" not in engine and "postgis" not in engine:
logger.warning("backup_database skipped — not PostgreSQL")
return
backup_dir = Path(settings.DB_BACKUP_LOCAL_DIR)
backup_dir.mkdir(parents=True, exist_ok=True)
date_str = datetime.now().strftime("%Y_%m_%d")
backup_path = backup_dir / f"vrobbler-backup-{date_str}.sql.gz"
env = os.environ.copy()
if db.get("PASSWORD"):
env["PGPASSWORD"] = db["PASSWORD"]
pg_dump_cmd = [
"pg_dump",
"--no-blobs",
"-h",
db.get("HOST", "localhost"),
"-U",
db.get("USER", "postgres"),
"-d",
db["NAME"],
]
logger.info("backup_database: dumping %s to %s", db["NAME"], backup_path)
try:
with open(backup_path, "wb") as f:
dump_proc = subprocess.Popen(pg_dump_cmd, stdout=subprocess.PIPE, env=env)
gzip_proc = subprocess.Popen(["gzip"], stdin=dump_proc.stdout, stdout=f)
dump_proc.stdout.close()
gzip_proc.communicate()
if gzip_proc.returncode != 0:
logger.error("backup_database: pg_dump / gzip failed")
_cleanup_failed_backup(backup_path)
return
dump_size = backup_path.stat().st_size
logger.info("backup_database: dump complete (%.1f MB)", dump_size / 1_000_000)
size_mb = dump_size / 1_000_000
locations = [str(backup_path)]
ssh_key = getattr(settings, "DB_BACKUP_SSH_KEY", "")
ssh_dest = getattr(settings, "DB_BACKUP_SSH_DEST", "")
cleanup_summary = None
if ssh_key and ssh_dest:
logger.info("backup_database: copying to %s", ssh_dest)
try:
subprocess.run(
["scp", "-i", ssh_key, str(backup_path), ssh_dest],
check=True,
capture_output=True,
text=True,
)
except subprocess.CalledProcessError as exc:
logger.error(
"backup_database: scp failed (stderr: %s)", exc.stderr.strip()
)
_cleanup_failed_backup(backup_path)
return
locations.append(ssh_dest)
logger.info("backup_database: copied to %s", ssh_dest)
# Parse user@host and path from dest
m = re.match(r"(\S+)@(\S+):(.+)", ssh_dest)
if m:
ssh_host = f"{m.group(1)}@{m.group(2)}"
remote_path = m.group(3)
logger.info("backup_database: pruning old remote backups")
cleanup_summary = _run_remote_cleanup(ssh_key, ssh_host, remote_path)
else:
logger.warning(
"backup_database: DB_BACKUP_SSH_KEY and DB_BACKUP_SSH_DEST not set — "
"backup saved locally at %s",
backup_path,
)
msg = (
f"✅ Vrobbler backup complete — {size_mb:.1f} MB\n"
f"Stored at: {', '.join(locations)}"
)
if cleanup_summary:
msg += f"\nRemote: {cleanup_summary}"
ntfy_url = getattr(
settings, "DB_BACKUP_NTFY_URL", "https://ntfy.unbl.ink/backups"
)
req.post(ntfy_url, data=msg.encode())
logger.info("backup_database: completed %s", backup_path)
except Exception as e:
logger.error("backup_database failed: %s", e)
_cleanup_failed_backup(backup_path)
ntfy_url = getattr(
settings, "DB_BACKUP_NTFY_URL", "https://ntfy.unbl.ink/backups"
)
req.post(ntfy_url, data=f"❌ Vrobbler backup FAILED: {e}".encode())
@shared_task
def import_from_lastfm_all_users():
"""Import Last.fm scrobbles for all users (replaces */30 cron)."""
from vrobbler.apps.scrobbles.utils import import_lastfm_for_all_users
import_lastfm_for_all_users()
@shared_task
def import_from_retroarch_all_users():
"""Import RetroArch scrobbles for all users (replaces @daily cron).
Deprecated: retroarch .lrtl files are now picked up by the WebDAV
importer (scan_webdav_for_retroarch). This task remains for manual use.
"""
import warnings
warnings.warn(
"import_from_retroarch_all_users is deprecated. "
"Upload .lrtl files to WebDAV var/retroarch/ instead.",
DeprecationWarning,
stacklevel=2,
)
from vrobbler.apps.scrobbles.utils import import_retroarch_for_all_users
import_retroarch_for_all_users()
@shared_task
def import_from_webdav_all_users():
"""Import from WebDAV for all users (replaces */2 cron)."""
from vrobbler.apps.scrobbles.importers.webdav import (
import_from_webdav_for_all_users,
)
import_from_webdav_for_all_users()
@shared_task
def import_from_imap_all_users():
"""Deprecated — BG Stats files now picked up from WebDAV var/bgstats/."""
import warnings
warnings.warn(
"IMAP import is deprecated. Upload .bgsplay files to WebDAV var/bgstats/ instead.",
DeprecationWarning,
stacklevel=2,
)
logger.warning("Skipping deprecated IMAP import (use WebDAV var/bgstats/ instead)")
@shared_task
def import_from_lichess_all_users():
"""Import chess games from Lichess for all users (replaces */20 cron)."""
from vrobbler.apps.boardgames.sources.lichess import (
import_chess_games_for_all_users,
)
import_chess_games_for_all_users()
@shared_task
def send_notification_for_in_progress():
"""Send ntfy stop-notifications for in-progress scrobbles (replaces */3 cron)."""
from vrobbler.apps.scrobbles.utils import (
send_stop_notifications_for_in_progress_scrobbles,
)
send_stop_notifications_for_in_progress_scrobbles()
@shared_task
def send_mood_checkin():
"""Send mood check-in reminders (replaces @hourly cron)."""
from vrobbler.apps.scrobbles.utils import send_mood_checkin_reminders
send_mood_checkin_reminders()
@shared_task
def backfill_scrobble_sentiment():
"""Backfill VADER sentiment for scrobbles with notes (replaces @hourly cron)."""
from scrobbles.models import Scrobble
from scrobbles.utils import analyze_scrobble_sentiment
qs = Scrobble.objects.filter(
models.Q(log__notes__isnull=False)
& ~models.Q(log__notes=[])
& ~models.Q(log__notes={})
& models.Q(log__sentiment__isnull=True)
)
count = 0
for scrobble in qs.iterator():
if analyze_scrobble_sentiment(scrobble):
count += 1
logger.info(
"Backfilled sentiment for %d scrobbles",
count,
)
@shared_task
def add_favorite_to_mopidy_playlist(favorite_id):
from scrobbles.models import FavoriteMedia
from scrobbles.utils import add_track_to_mopidy_favorites_playlist
favorite = FavoriteMedia.objects.filter(id=favorite_id).first()
if not favorite:
return
add_track_to_mopidy_favorites_playlist(favorite)
@shared_task
def remove_favorite_from_mopidy_playlist(user_id, track_id):
from music.models import Track
from scrobbles.utils import remove_track_from_mopidy_favorites_playlist
User = get_user_model()
try:
user = User.objects.get(id=user_id)
track = Track.objects.get(id=track_id)
except (User.DoesNotExist, Track.DoesNotExist):
return
import types
proxy = types.SimpleNamespace(
media_type="Track",
user=user,
user_id=user.id,
track=track,
)
remove_track_from_mopidy_favorites_playlist(proxy)
@shared_task
def add_scrobble_to_mopidy_queue(scrobble_id):
from scrobbles.models import Scrobble
scrobble = Scrobble.objects.filter(id=scrobble_id).first()
if not scrobble:
return
profile = scrobble.user.profile
mopidy_url = profile.mopidy_api_url
if not mopidy_url:
return
mopidy_uri = (scrobble.log or {}).get("raw_data", {}).get("mopidy_uri")
track = scrobble.track if scrobble.media_type == "Track" else None
if not mopidy_uri and track:
sibling = (
Scrobble.objects.filter(track=track, user=scrobble.user)
.order_by("-timestamp")
.iterator()
)
for s in sibling:
uri = (s.log or {}).get("raw_data", {}).get("mopidy_uri")
if uri:
mopidy_uri = uri
break
if not mopidy_uri:
logger.warning(
"No Mopidy URI found for scrobble",
extra={"scrobble_id": scrobble_id, "user_id": scrobble.user_id},
)
return
import requests
rpc_url = mopidy_url.rstrip("/") + "/rpc"
payload = {
"jsonrpc": "2.0",
"id": 1,
"method": "core.tracklist.add",
"params": {"uris": [mopidy_uri]},
}
try:
resp = requests.post(rpc_url, json=payload, timeout=10)
resp.raise_for_status()
rpc_result = resp.json()
if rpc_result.get("error"):
logger.error(
"Mopidy error adding to queue",
extra={"error": rpc_result["error"], "scrobble_id": scrobble_id},
)
else:
logger.info(
"Added track to Mopidy queue",
extra={"scrobble_id": scrobble_id, "mopidy_uri": mopidy_uri},
)
except requests.RequestException as e:
logger.error(
"Failed to add track to Mopidy queue",
extra={"scrobble_id": scrobble_id, "error": str(e)},
)
@shared_task
def add_scrobble_to_mopidy_monthly_playlist(scrobble_id):
from scrobbles.models import Scrobble
from scrobbles.utils import add_track_to_mopidy_monthly_playlist
scrobble = Scrobble.objects.filter(id=scrobble_id).first()
if not scrobble:
return
track = scrobble.track if scrobble.media_type == "Track" else None
if track:
sibling = (
Scrobble.objects.filter(track=track, user=scrobble.user)
.order_by("-timestamp")
.iterator()
)
for s in sibling:
if (s.log or {}).get("raw_data", {}).get("mopidy_uri"):
scrobble = s
break
add_track_to_mopidy_monthly_playlist(scrobble)