[tasks] Add more visibility to backups

README.md

@@ -21,3 +21,22 @@ VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True
 VROBBLER_DATABASE_URL="postgres://vrobbler:<pass>@db.service:5432/vrobbler"
 VROBBLER_REDIS_URL="redis://:<pass>@cache.service:6379/0"
 ```
+
+## Database Backup
+
+A backup command is available via `./manage.py backup_database` (also runs on a cron schedule via Celery). It dumps the database with `pg_dump`, compresses with gzip, and optionally copies the backup to a remote host via SCP.
+
+Configure these additional settings as needed:
+
+```
+DB_BACKUP_SSH_KEY="/path/to/ssh/private/key"
+DB_BACKUP_SSH_DEST="user@backup.example.com:/remote/path/"
+DB_BACKUP_NTFY_URL="https://ntfy.sh/your-topic"
+```
+
+- `DB_BACKUP_SSH_KEY` — Path to the SSH private key used for remote copy.
+- `DB_BACKUP_SSH_DEST` — SCP destination (user@host:path). If set alongside SSH_KEY, the backup is copied to the remote host and old backups are pruned.
+- `DB_BACKUP_NTFY_URL` — ntfy.sh URL for success notifications. Defaults to `https://ntfy.unbl.ink/backups`.
+
+Retention is hardcoded: keeps daily backups for 7 days, plus one per month for 12 months.
+```

vrobbler/apps/scrobbles/management/commands/backup_database.py

@@ -0,0 +1,7 @@
+from django.core.management.base import BaseCommand
+from vrobbler.apps.scrobbles.tasks import backup_database
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **options):
+        backup_database()

vrobbler/apps/scrobbles/tasks.py

@@ -257,6 +257,16 @@ BACKUP_RETENTION_DAYS = 7
 BACKUP_RETENTION_MONTHS = 12
 
 
+def _cleanup_failed_backup(backup_path):
+    """Remove a failed/incomplete backup file if it exists."""
+    from pathlib import Path
+
+    p = Path(backup_path)
+    if p.exists():
+        p.unlink()
+        logger.warning("backup_database: removed incomplete backup %s", backup_path)
+
+
 def _retention_files_to_delete(remote_files, now):
     """Return list of filenames to delete under retention policy.
 
@@ -353,8 +363,10 @@ def backup_database():
         logger.warning("backup_database skipped — not PostgreSQL")
         return
 
+    backup_dir = Path("/var/backup/vrobbler")
+    backup_dir.mkdir(parents=True, exist_ok=True)
     date_str = datetime.now().strftime("%Y_%m_%d")
-    backup_path = Path(f"/tmp/vrobbler-backup-{date_str}.sql.gz")
+    backup_path = backup_dir / f"vrobbler-backup-{date_str}.sql.gz"
 
     env = os.environ.copy()
     if db.get("PASSWORD"):
@@ -368,6 +380,8 @@ def backup_database():
         "-d", db["NAME"],
     ]
 
+    logger.info("backup_database: dumping %s to %s", db["NAME"], backup_path)
+
     try:
         with open(backup_path, "wb") as f:
             dump_proc = subprocess.Popen(pg_dump_cmd, stdout=subprocess.PIPE, env=env)
@@ -379,15 +393,31 @@ def backup_database():
 
         if gzip_proc.returncode != 0:
             logger.error("backup_database: pg_dump / gzip failed")
+            _cleanup_failed_backup(backup_path)
             return
 
+        dump_size = backup_path.stat().st_size
+        logger.info(
+            "backup_database: dump complete (%.1f MB)", dump_size / 1_000_000
+        )
+
         ssh_key = getattr(settings, "DB_BACKUP_SSH_KEY", "")
         ssh_dest = getattr(settings, "DB_BACKUP_SSH_DEST", "")
         if ssh_key and ssh_dest:
-            subprocess.run(
-                ["scp", "-i", ssh_key, str(backup_path), ssh_dest],
-                check=True,
-            )
+            logger.info("backup_database: copying to %s", ssh_dest)
+            try:
+                subprocess.run(
+                    ["scp", "-i", ssh_key, str(backup_path), ssh_dest],
+                    check=True,
+                    capture_output=True,
+                    text=True,
+                )
+            except subprocess.CalledProcessError as exc:
+                logger.error(
+                    "backup_database: scp failed (stderr: %s)", exc.stderr.strip()
+                )
+                _cleanup_failed_backup(backup_path)
+                return
             logger.info("backup_database: copied to %s", ssh_dest)
 
             # Parse user@host and path from dest
@@ -395,17 +425,23 @@ def backup_database():
             if m:
                 ssh_host = f"{m.group(1)}@{m.group(2)}"
                 remote_path = m.group(3)
+                logger.info("backup_database: pruning old remote backups")
                 _run_remote_cleanup(ssh_key, ssh_host, remote_path)
+        else:
+            logger.warning(
+                "backup_database: DB_BACKUP_SSH_KEY and DB_BACKUP_SSH_DEST not set — "
+                "backup saved locally at %s",
+                backup_path,
+            )
 
         ntfy_url = getattr(
             settings, "DB_BACKUP_NTFY_URL", "https://ntfy.unbl.ink/backups"
         )
         req.post(ntfy_url, data=b"Vrobbler backup succeeded")
         logger.info("backup_database: completed %s", backup_path)
-
-        backup_path.unlink(missing_ok=True)
     except Exception as e:
         logger.error("backup_database failed: %s", e)
+        _cleanup_failed_backup(backup_path)
 
 
 @shared_task