[metadata] Fix cleanup scripts to check for dead images

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [0/23] :vrobbler:project:personal:
+* Backlog [1/24] :vrobbler:project:personal:
 ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles:
 :PROPERTIES:
@@ -605,6 +605,10 @@ independent of the email flow it was originally creatdd for
 
 ** TODO [#B] Is there way to create unique slugs for media instances :media_types:
 
+** DONE [#B] Cleanup commands should check for broken images :metadata:cleanup:
+:PROPERTIES:
+:ID:       bacce321-73c7-ae1f-bfa7-c3ee517b5441
+:END:
 * Version 58.5 [1/1]
 ** DONE [#A] The maloja style charts are messed up :templates:charts:
 :PROPERTIES:

vrobbler/apps/books/management/commands/cleanup_book_metadata.py

@@ -18,8 +18,17 @@ MISSING_ALL = [
     "publish_year",
 ]
 
+def _cover_missing_or_broken(book) -> bool:
+    if not bool(book.cover):
+        return True
+    try:
+        return not book.cover.storage.exists(book.cover.name)
+    except Exception:
+        return True
+
+
 MISSING_GROUPS = {
-    "cover": lambda b: not bool(b.cover),
+    "cover": _cover_missing_or_broken,
     "summary": lambda b: not b.summary,
     "isbn": lambda b: not b.isbn_13 and not b.isbn_10,
     "pages": lambda b: b.pages is None,

vrobbler/apps/music/management/commands/cleanup_music_metadata.py

@@ -0,0 +1,202 @@
+import logging
+
+from django.core.management.base import BaseCommand
+from django.db import models, transaction
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Enrich artist and album metadata (covers, thumbnails) from MusicBrainz and TheAudioDB"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--force",
+            action="store_true",
+            help="Overwrite existing cover image and metadata",
+        )
+        parser.add_argument(
+            "--dry-run",
+            action="store_true",
+            help="Show what would be done without making changes",
+        )
+        parser.add_argument(
+            "--artists",
+            action="store_true",
+            help="Only process artists",
+        )
+        parser.add_argument(
+            "--albums",
+            action="store_true",
+            help="Only process albums",
+        )
+        parser.add_argument(
+            "--needs-metadata",
+            action="store_true",
+            help="Only process items missing metadata or with broken images",
+        )
+
+    def _has_broken_image(self, obj, field_name: str) -> bool:
+        field = getattr(obj, field_name, None)
+        if not field or not field.name:
+            return False
+        try:
+            return not field.storage.exists(field.name)
+        except Exception:
+            return True
+
+    def handle(self, *args, **options):
+        from music.models import Album, Artist
+
+        force = options["force"]
+        dry_run = options["dry_run"]
+        only_artists = options["artists"]
+        only_albums = options["albums"]
+        needs_metadata = options["needs_metadata"]
+
+        if not only_artists and not only_albums:
+            only_artists = only_albums = True
+
+        updated_total = 0
+        errors_total = 0
+
+        if only_artists:
+            updated_total += self._process_artists(force, dry_run, needs_metadata)
+        errors_total = 0  # reset per section
+
+        if only_albums:
+            updated_total += self._process_albums(force, dry_run, needs_metadata)
+
+        self.stdout.write(
+            self.style.SUCCESS(f"\nDone! {updated_total} items processed")
+        )
+
+    def _get_artists(self, needs_metadata: bool):
+        from music.models import Artist
+
+        qs = Artist.objects.all()
+        if needs_metadata:
+            qs = qs.filter(
+                models.Q(theaudiodb_id__isnull=True)
+                | models.Q(theaudiodb_id="")
+                | models.Q(thumbnail__isnull=True)
+                | models.Q(thumbnail="")
+            )
+
+        broken = []
+        if needs_metadata:
+            broken_qs = Artist.objects.exclude(
+                models.Q(thumbnail__isnull=True) | models.Q(thumbnail=""),
+            )
+            for artist in broken_qs.iterator():
+                if self._has_broken_image(artist, "thumbnail"):
+                    broken.append(artist)
+
+        return list(qs) + broken
+
+    def _get_albums(self, needs_metadata: bool):
+        from music.models import Album
+
+        qs = Album.objects.all()
+        if needs_metadata:
+            qs = qs.filter(
+                models.Q(cover_image__isnull=True)
+                | models.Q(cover_image="")
+                | models.Q(cover_image="default-image-replace-me")
+            )
+
+        broken = []
+        if needs_metadata:
+            broken_qs = Album.objects.exclude(
+                models.Q(cover_image__isnull=True) | models.Q(cover_image=""),
+            )
+            for album in broken_qs.iterator():
+                if self._has_broken_image(album, "cover_image"):
+                    broken.append(album)
+
+        return list(qs) + broken
+
+    def _process_artists(self, force, dry_run, needs_metadata):
+        from music.models import Artist
+
+        artists = self._get_artists(needs_metadata) if needs_metadata else list(Artist.objects.all())
+        total = len(artists)
+        self.stdout.write(f"Processing {total} artists")
+
+        if dry_run:
+            for artist in artists:
+                has_tadb = bool(artist.theaudiodb_id)
+                has_thumb = bool(artist.thumbnail)
+                thumb_broken = self._has_broken_image(artist, "thumbnail")
+                status = f"theaudiodb_id={'✓' if has_tadb else '✗'}"
+                if thumb_broken:
+                    status += ", thumbnail=BROKEN"
+                elif has_thumb:
+                    status += ", thumbnail=✓"
+                else:
+                    status += ", thumbnail=✗"
+                self.stdout.write(f"  [DRY RUN] Would fix {artist.name} ({status})")
+            return 0
+
+        updated = 0
+        errors = 0
+        for artist in artists:
+            try:
+                with transaction.atomic():
+                    artist.fix_metadata(
+                        force_update=force or self._has_broken_image(artist, "thumbnail")
+                    )
+                updated += 1
+                self.stdout.write(f"  [ARTIST {updated}/{total}] {artist.name}")
+            except Exception as e:
+                errors += 1
+                self.stdout.write(
+                    self.style.ERROR(f"  Error updating artist {artist.name}: {e}")
+                )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"\nArtists done! {updated} updated, {errors} errors")
+        )
+        return updated
+
+    def _process_albums(self, force, dry_run, needs_metadata):
+        from music.models import Album
+
+        albums = self._get_albums(needs_metadata) if needs_metadata else list(Album.objects.all())
+        total = len(albums)
+        self.stdout.write(f"Processing {total} albums")
+
+        if dry_run:
+            for album in albums:
+                has_cover = bool(album.cover_image)
+                cover_broken = self._has_broken_image(album, "cover_image")
+                if cover_broken:
+                    status = "cover=BROKEN"
+                elif has_cover:
+                    status = "cover=✓"
+                else:
+                    status = "cover=✗"
+                self.stdout.write(f"  [DRY RUN] Would fix {album.name} ({status})")
+            return 0
+
+        updated = 0
+        errors = 0
+        for album in albums:
+            try:
+                with transaction.atomic():
+                    if self._has_broken_image(album, "cover_image") or force:
+                        album.fetch_artwork(force=True)
+                    else:
+                        album.fix_metadata()
+                updated += 1
+                self.stdout.write(f"  [ALBUM {updated}/{total}] {album.name}")
+            except Exception as e:
+                errors += 1
+                self.stdout.write(
+                    self.style.ERROR(f"  Error updating album {album.name}: {e}")
+                )
+
+        self.stdout.write(
+            self.style.SUCCESS(f"\nAlbums done! {updated} updated, {errors} errors")
+        )
+        return updated

vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py

@@ -17,8 +17,8 @@ MISSING_ALL = [
 ]
 
 MISSING_GROUPS = {
-    "cover": lambda g: not bool(g.cover),
-    "screenshot": lambda g: not bool(g.screenshot),
+    "cover": lambda g: _image_missing_or_broken(g, "cover"),
+    "screenshot": lambda g: _image_missing_or_broken(g, "screenshot"),
     "summary": lambda g: not g.summary,
     "rating": lambda g: g.rating is None,
     "release_date": lambda g: g.release_date is None,
@@ -28,6 +28,16 @@ MISSING_GROUPS = {
 }
 
 
+def _image_missing_or_broken(game, field_name) -> bool:
+    field = getattr(game, field_name)
+    if not bool(field):
+        return True
+    try:
+        return not field.storage.exists(field.name)
+    except Exception:
+        return True
+
+
 def _game_matches(game, flags):
     if not flags:
         return False
@@ -103,6 +113,7 @@ class Command(BaseCommand):
 
         if all_missing:
             flags = MISSING_ALL
+            fix_broken_images = True
 
         if not flags and not game_id and not force and not fix_broken_images:
             self.stdout.write(

vrobbler/apps/videos/management/commands/cleanup_channel_metadata.py

@@ -30,6 +30,19 @@ class Command(BaseCommand):
             action="store_true",
             help="Only process channels with a twitch_id",
         )
+        parser.add_argument(
+            "--needs-metadata",
+            action="store_true",
+            help="Only process channels missing youtube_id, twitch_id, cover image, or with broken cover image",
+        )
+
+    def _has_broken_image(self, channel) -> bool:
+        if not channel.cover_image or not channel.cover_image.name:
+            return False
+        try:
+            return not channel.cover_image.storage.exists(channel.cover_image.name)
+        except Exception:
+            return True
 
     def handle(self, *args, **options):
         from videos.models import Channel
@@ -38,6 +51,7 @@ class Command(BaseCommand):
         dry_run = options["dry_run"]
         youtube_only = options["youtube_only"]
         twitch_only = options["twitch_only"]
+        needs_metadata = options["needs_metadata"]
 
         qs = Channel.objects.all()
 
@@ -45,29 +59,61 @@ class Command(BaseCommand):
             qs = qs.exclude(youtube_id__isnull=True).exclude(youtube_id="")
         elif twitch_only:
             qs = qs.exclude(twitch_id__isnull=True).exclude(twitch_id="")
+        elif needs_metadata:
+            no_id = models.Q(youtube_id__isnull=True) & models.Q(twitch_id__isnull=True)
+            no_id |= models.Q(youtube_id="") & models.Q(twitch_id="")
+            no_id |= models.Q(youtube_id__isnull=True) & models.Q(twitch_id="")
+            no_id |= models.Q(youtube_id="") & models.Q(twitch_id__isnull=True)
+            qs = qs.filter(
+                no_id
+                | models.Q(cover_image__isnull=True)
+                | models.Q(cover_image="")
+            )
         else:
             qs = qs.filter(
                 models.Q(youtube_id__isnull=False) | models.Q(twitch_id__isnull=False)
             ).exclude(youtube_id="", twitch_id="")
 
         total = qs.count()
-        self.stdout.write(f"Processing {total} channels")
+        self.stdout.write(f"Processing {total} channels from DB filter")
+
+        broken_channels = []
+        if needs_metadata:
+            broken_qs = Channel.objects.filter(
+                cover_image__isnull=False,
+            ).exclude(
+                cover_image="",
+            )
+            if youtube_only:
+                broken_qs = broken_qs.exclude(youtube_id__isnull=True).exclude(youtube_id="")
+            elif twitch_only:
+                broken_qs = broken_qs.exclude(twitch_id__isnull=True).exclude(twitch_id="")
+            for channel in broken_qs.iterator():
+                if self._has_broken_image(channel):
+                    broken_channels.append(channel)
+
+        all_channels = list(qs) + broken_channels
+        total = len(all_channels)
+        self.stdout.write(f"Total channels to process: {total}")
 
         if dry_run:
-            for channel in qs.iterator():
+            for channel in all_channels:
                 source = "youtube" if channel.youtube_id else "twitch"
                 identifier = channel.youtube_id or channel.twitch_id
+                status = f"({source}: {identifier})"
+                if self._has_broken_image(channel):
+                    status += " [image BROKEN]"
                 self.stdout.write(
-                    f"  [DRY RUN] Would fix {channel.name} ({source}: {identifier})"
+                    f"  [DRY RUN] Would fix {channel.name} {status}"
                 )
             return
 
         updated = 0
         errors = 0
-        for channel in qs.iterator():
+        for channel in all_channels:
             try:
                 with transaction.atomic():
-                    channel.fix_metadata(force=force)
+                    channel.fix_metadata(force=force or self._has_broken_image(channel))
                 updated += 1
                 source = "youtube" if channel.youtube_id else "twitch"
                 self.stdout.write(f"  [{updated}/{total}] {channel.name} ({source})")

vrobbler/apps/videos/management/commands/cleanup_series_metadata.py

@@ -1,4 +1,5 @@
 import logging
+import os
 
 from django.core.management.base import BaseCommand
 from django.db import models, transaction
@@ -28,9 +29,18 @@ class Command(BaseCommand):
         parser.add_argument(
             "--needs-metadata",
             action="store_true",
-            help="Only process series missing imdb_id or cover image",
+            help="Only process series missing imdb_id or with broken cover image",
         )
 
+    def _has_broken_image(self, series) -> bool:
+        """Check if a series has a cover_image set but the file is missing."""
+        if not series.cover_image:
+            return False
+        try:
+            return not os.path.exists(series.cover_image.path)
+        except Exception:
+            return True
+
     def handle(self, *args, **options):
         from videos.models import Series
 
@@ -51,25 +61,50 @@ class Command(BaseCommand):
             )
 
         total = qs.count()
-        self.stdout.write(f"Processing {total} series")
+        self.stdout.write(f"Processing {total} series from DB filter")
+
+        # Also find series with broken cover images
+        broken_image_series = []
+        if needs_metadata:
+            broken_qs = Series.objects.filter(
+                cover_image__isnull=False,
+            ).exclude(
+                models.Q(imdb_id__isnull=True)
+                | models.Q(imdb_id="")
+                | models.Q(cover_image__isnull=True)
+                | models.Q(cover_image=""),
+            )
+            if imdb_id:
+                broken_qs = broken_qs.filter(imdb_id=imdb_id)
+            for series in broken_qs.iterator():
+                if self._has_broken_image(series):
+                    broken_image_series.append(series)
+
+        all_series = list(qs) + broken_image_series
+        total = len(all_series)
+        self.stdout.write(f"Total series to process: {total}")
 
         if dry_run:
-            for series in qs.iterator():
+            for series in all_series:
                 has_imdb = bool(series.imdb_id)
                 has_image = bool(series.cover_image)
-                self.stdout.write(
-                    f"  [DRY RUN] Would fix {series.name}"
-                    f" (imdb_id={'✓' if has_imdb else '✗'}"
-                    f", image={'✓' if has_image else '✗'})"
-                )
+                image_broken = self._has_broken_image(series)
+                status = f"imdb_id={'✓' if has_imdb else '✗'}"
+                if image_broken:
+                    status += ", image=BROKEN"
+                elif has_image:
+                    status += ", image=✓"
+                else:
+                    status += ", image=✗"
+                self.stdout.write(f"  [DRY RUN] Would fix {series.name} ({status})")
             return
 
         updated = 0
         errors = 0
-        for series in qs.iterator():
+        for series in all_series:
             try:
                 with transaction.atomic():
-                    series.fix_metadata(force_update=force)
+                    series.fix_metadata(force_update=force or self._has_broken_image(series))
                 updated += 1
                 self.stdout.write(f"  [{updated}/{total}] {series.name}")
             except Exception as e: