From cb23d5a5be20e418403f1b0444611cfc2138d8db Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Tue, 30 Jun 2026 16:03:29 -0400 Subject: [PATCH] [metadata] Fix cleanup scripts to check for dead images --- PROJECT.org | 6 +- .../commands/cleanup_book_metadata.py | 11 +- .../commands/cleanup_music_metadata.py | 202 ++++++++++++++++++ .../commands/cleanup_videogame_metadata.py | 15 +- .../commands/cleanup_channel_metadata.py | 56 ++++- .../commands/cleanup_series_metadata.py | 55 ++++- 6 files changed, 326 insertions(+), 19 deletions(-) create mode 100644 vrobbler/apps/music/management/commands/cleanup_music_metadata.py diff --git a/PROJECT.org b/PROJECT.org index 10a0850..07a21c2 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/23] :vrobbler:project:personal: +* Backlog [1/24] :vrobbler:project:personal: ** TODO [#C] After transition to linux add curl_cffi as webpage scrapper again :webpages:metadata: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: @@ -605,6 +605,10 @@ independent of the email flow it was originally creatdd for ** TODO [#B] Is there way to create unique slugs for media instances :media_types: +** DONE [#B] Cleanup commands should check for broken images :metadata:cleanup: +:PROPERTIES: +:ID: bacce321-73c7-ae1f-bfa7-c3ee517b5441 +:END: * Version 58.5 [1/1] ** DONE [#A] The maloja style charts are messed up :templates:charts: :PROPERTIES: diff --git a/vrobbler/apps/books/management/commands/cleanup_book_metadata.py b/vrobbler/apps/books/management/commands/cleanup_book_metadata.py index 7ba5959..851db14 100644 --- a/vrobbler/apps/books/management/commands/cleanup_book_metadata.py +++ b/vrobbler/apps/books/management/commands/cleanup_book_metadata.py @@ -18,8 +18,17 @@ MISSING_ALL = [ "publish_year", ] +def _cover_missing_or_broken(book) -> bool: + if not bool(book.cover): + return True + try: + return not book.cover.storage.exists(book.cover.name) + except Exception: + return True + + MISSING_GROUPS = { - "cover": lambda b: not bool(b.cover), + "cover": _cover_missing_or_broken, "summary": lambda b: not b.summary, "isbn": lambda b: not b.isbn_13 and not b.isbn_10, "pages": lambda b: b.pages is None, diff --git a/vrobbler/apps/music/management/commands/cleanup_music_metadata.py b/vrobbler/apps/music/management/commands/cleanup_music_metadata.py new file mode 100644 index 0000000..e939238 --- /dev/null +++ b/vrobbler/apps/music/management/commands/cleanup_music_metadata.py @@ -0,0 +1,202 @@ +import logging + +from django.core.management.base import BaseCommand +from django.db import models, transaction + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Enrich artist and album metadata (covers, thumbnails) from MusicBrainz and TheAudioDB" + + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Overwrite existing cover image and metadata", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without making changes", + ) + parser.add_argument( + "--artists", + action="store_true", + help="Only process artists", + ) + parser.add_argument( + "--albums", + action="store_true", + help="Only process albums", + ) + parser.add_argument( + "--needs-metadata", + action="store_true", + help="Only process items missing metadata or with broken images", + ) + + def _has_broken_image(self, obj, field_name: str) -> bool: + field = getattr(obj, field_name, None) + if not field or not field.name: + return False + try: + return not field.storage.exists(field.name) + except Exception: + return True + + def handle(self, *args, **options): + from music.models import Album, Artist + + force = options["force"] + dry_run = options["dry_run"] + only_artists = options["artists"] + only_albums = options["albums"] + needs_metadata = options["needs_metadata"] + + if not only_artists and not only_albums: + only_artists = only_albums = True + + updated_total = 0 + errors_total = 0 + + if only_artists: + updated_total += self._process_artists(force, dry_run, needs_metadata) + errors_total = 0 # reset per section + + if only_albums: + updated_total += self._process_albums(force, dry_run, needs_metadata) + + self.stdout.write( + self.style.SUCCESS(f"\nDone! {updated_total} items processed") + ) + + def _get_artists(self, needs_metadata: bool): + from music.models import Artist + + qs = Artist.objects.all() + if needs_metadata: + qs = qs.filter( + models.Q(theaudiodb_id__isnull=True) + | models.Q(theaudiodb_id="") + | models.Q(thumbnail__isnull=True) + | models.Q(thumbnail="") + ) + + broken = [] + if needs_metadata: + broken_qs = Artist.objects.exclude( + models.Q(thumbnail__isnull=True) | models.Q(thumbnail=""), + ) + for artist in broken_qs.iterator(): + if self._has_broken_image(artist, "thumbnail"): + broken.append(artist) + + return list(qs) + broken + + def _get_albums(self, needs_metadata: bool): + from music.models import Album + + qs = Album.objects.all() + if needs_metadata: + qs = qs.filter( + models.Q(cover_image__isnull=True) + | models.Q(cover_image="") + | models.Q(cover_image="default-image-replace-me") + ) + + broken = [] + if needs_metadata: + broken_qs = Album.objects.exclude( + models.Q(cover_image__isnull=True) | models.Q(cover_image=""), + ) + for album in broken_qs.iterator(): + if self._has_broken_image(album, "cover_image"): + broken.append(album) + + return list(qs) + broken + + def _process_artists(self, force, dry_run, needs_metadata): + from music.models import Artist + + artists = self._get_artists(needs_metadata) if needs_metadata else list(Artist.objects.all()) + total = len(artists) + self.stdout.write(f"Processing {total} artists") + + if dry_run: + for artist in artists: + has_tadb = bool(artist.theaudiodb_id) + has_thumb = bool(artist.thumbnail) + thumb_broken = self._has_broken_image(artist, "thumbnail") + status = f"theaudiodb_id={'✓' if has_tadb else '✗'}" + if thumb_broken: + status += ", thumbnail=BROKEN" + elif has_thumb: + status += ", thumbnail=✓" + else: + status += ", thumbnail=✗" + self.stdout.write(f" [DRY RUN] Would fix {artist.name} ({status})") + return 0 + + updated = 0 + errors = 0 + for artist in artists: + try: + with transaction.atomic(): + artist.fix_metadata( + force_update=force or self._has_broken_image(artist, "thumbnail") + ) + updated += 1 + self.stdout.write(f" [ARTIST {updated}/{total}] {artist.name}") + except Exception as e: + errors += 1 + self.stdout.write( + self.style.ERROR(f" Error updating artist {artist.name}: {e}") + ) + + self.stdout.write( + self.style.SUCCESS(f"\nArtists done! {updated} updated, {errors} errors") + ) + return updated + + def _process_albums(self, force, dry_run, needs_metadata): + from music.models import Album + + albums = self._get_albums(needs_metadata) if needs_metadata else list(Album.objects.all()) + total = len(albums) + self.stdout.write(f"Processing {total} albums") + + if dry_run: + for album in albums: + has_cover = bool(album.cover_image) + cover_broken = self._has_broken_image(album, "cover_image") + if cover_broken: + status = "cover=BROKEN" + elif has_cover: + status = "cover=✓" + else: + status = "cover=✗" + self.stdout.write(f" [DRY RUN] Would fix {album.name} ({status})") + return 0 + + updated = 0 + errors = 0 + for album in albums: + try: + with transaction.atomic(): + if self._has_broken_image(album, "cover_image") or force: + album.fetch_artwork(force=True) + else: + album.fix_metadata() + updated += 1 + self.stdout.write(f" [ALBUM {updated}/{total}] {album.name}") + except Exception as e: + errors += 1 + self.stdout.write( + self.style.ERROR(f" Error updating album {album.name}: {e}") + ) + + self.stdout.write( + self.style.SUCCESS(f"\nAlbums done! {updated} updated, {errors} errors") + ) + return updated diff --git a/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py b/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py index 5996552..427bf55 100644 --- a/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py +++ b/vrobbler/apps/videogames/management/commands/cleanup_videogame_metadata.py @@ -17,8 +17,8 @@ MISSING_ALL = [ ] MISSING_GROUPS = { - "cover": lambda g: not bool(g.cover), - "screenshot": lambda g: not bool(g.screenshot), + "cover": lambda g: _image_missing_or_broken(g, "cover"), + "screenshot": lambda g: _image_missing_or_broken(g, "screenshot"), "summary": lambda g: not g.summary, "rating": lambda g: g.rating is None, "release_date": lambda g: g.release_date is None, @@ -28,6 +28,16 @@ MISSING_GROUPS = { } +def _image_missing_or_broken(game, field_name) -> bool: + field = getattr(game, field_name) + if not bool(field): + return True + try: + return not field.storage.exists(field.name) + except Exception: + return True + + def _game_matches(game, flags): if not flags: return False @@ -103,6 +113,7 @@ class Command(BaseCommand): if all_missing: flags = MISSING_ALL + fix_broken_images = True if not flags and not game_id and not force and not fix_broken_images: self.stdout.write( diff --git a/vrobbler/apps/videos/management/commands/cleanup_channel_metadata.py b/vrobbler/apps/videos/management/commands/cleanup_channel_metadata.py index bcf4668..1e46cb8 100644 --- a/vrobbler/apps/videos/management/commands/cleanup_channel_metadata.py +++ b/vrobbler/apps/videos/management/commands/cleanup_channel_metadata.py @@ -30,6 +30,19 @@ class Command(BaseCommand): action="store_true", help="Only process channels with a twitch_id", ) + parser.add_argument( + "--needs-metadata", + action="store_true", + help="Only process channels missing youtube_id, twitch_id, cover image, or with broken cover image", + ) + + def _has_broken_image(self, channel) -> bool: + if not channel.cover_image or not channel.cover_image.name: + return False + try: + return not channel.cover_image.storage.exists(channel.cover_image.name) + except Exception: + return True def handle(self, *args, **options): from videos.models import Channel @@ -38,6 +51,7 @@ class Command(BaseCommand): dry_run = options["dry_run"] youtube_only = options["youtube_only"] twitch_only = options["twitch_only"] + needs_metadata = options["needs_metadata"] qs = Channel.objects.all() @@ -45,29 +59,61 @@ class Command(BaseCommand): qs = qs.exclude(youtube_id__isnull=True).exclude(youtube_id="") elif twitch_only: qs = qs.exclude(twitch_id__isnull=True).exclude(twitch_id="") + elif needs_metadata: + no_id = models.Q(youtube_id__isnull=True) & models.Q(twitch_id__isnull=True) + no_id |= models.Q(youtube_id="") & models.Q(twitch_id="") + no_id |= models.Q(youtube_id__isnull=True) & models.Q(twitch_id="") + no_id |= models.Q(youtube_id="") & models.Q(twitch_id__isnull=True) + qs = qs.filter( + no_id + | models.Q(cover_image__isnull=True) + | models.Q(cover_image="") + ) else: qs = qs.filter( models.Q(youtube_id__isnull=False) | models.Q(twitch_id__isnull=False) ).exclude(youtube_id="", twitch_id="") total = qs.count() - self.stdout.write(f"Processing {total} channels") + self.stdout.write(f"Processing {total} channels from DB filter") + + broken_channels = [] + if needs_metadata: + broken_qs = Channel.objects.filter( + cover_image__isnull=False, + ).exclude( + cover_image="", + ) + if youtube_only: + broken_qs = broken_qs.exclude(youtube_id__isnull=True).exclude(youtube_id="") + elif twitch_only: + broken_qs = broken_qs.exclude(twitch_id__isnull=True).exclude(twitch_id="") + for channel in broken_qs.iterator(): + if self._has_broken_image(channel): + broken_channels.append(channel) + + all_channels = list(qs) + broken_channels + total = len(all_channels) + self.stdout.write(f"Total channels to process: {total}") if dry_run: - for channel in qs.iterator(): + for channel in all_channels: source = "youtube" if channel.youtube_id else "twitch" identifier = channel.youtube_id or channel.twitch_id + status = f"({source}: {identifier})" + if self._has_broken_image(channel): + status += " [image BROKEN]" self.stdout.write( - f" [DRY RUN] Would fix {channel.name} ({source}: {identifier})" + f" [DRY RUN] Would fix {channel.name} {status}" ) return updated = 0 errors = 0 - for channel in qs.iterator(): + for channel in all_channels: try: with transaction.atomic(): - channel.fix_metadata(force=force) + channel.fix_metadata(force=force or self._has_broken_image(channel)) updated += 1 source = "youtube" if channel.youtube_id else "twitch" self.stdout.write(f" [{updated}/{total}] {channel.name} ({source})") diff --git a/vrobbler/apps/videos/management/commands/cleanup_series_metadata.py b/vrobbler/apps/videos/management/commands/cleanup_series_metadata.py index b70659c..eba6797 100644 --- a/vrobbler/apps/videos/management/commands/cleanup_series_metadata.py +++ b/vrobbler/apps/videos/management/commands/cleanup_series_metadata.py @@ -1,4 +1,5 @@ import logging +import os from django.core.management.base import BaseCommand from django.db import models, transaction @@ -28,9 +29,18 @@ class Command(BaseCommand): parser.add_argument( "--needs-metadata", action="store_true", - help="Only process series missing imdb_id or cover image", + help="Only process series missing imdb_id or with broken cover image", ) + def _has_broken_image(self, series) -> bool: + """Check if a series has a cover_image set but the file is missing.""" + if not series.cover_image: + return False + try: + return not os.path.exists(series.cover_image.path) + except Exception: + return True + def handle(self, *args, **options): from videos.models import Series @@ -51,25 +61,50 @@ class Command(BaseCommand): ) total = qs.count() - self.stdout.write(f"Processing {total} series") + self.stdout.write(f"Processing {total} series from DB filter") + + # Also find series with broken cover images + broken_image_series = [] + if needs_metadata: + broken_qs = Series.objects.filter( + cover_image__isnull=False, + ).exclude( + models.Q(imdb_id__isnull=True) + | models.Q(imdb_id="") + | models.Q(cover_image__isnull=True) + | models.Q(cover_image=""), + ) + if imdb_id: + broken_qs = broken_qs.filter(imdb_id=imdb_id) + for series in broken_qs.iterator(): + if self._has_broken_image(series): + broken_image_series.append(series) + + all_series = list(qs) + broken_image_series + total = len(all_series) + self.stdout.write(f"Total series to process: {total}") if dry_run: - for series in qs.iterator(): + for series in all_series: has_imdb = bool(series.imdb_id) has_image = bool(series.cover_image) - self.stdout.write( - f" [DRY RUN] Would fix {series.name}" - f" (imdb_id={'✓' if has_imdb else '✗'}" - f", image={'✓' if has_image else '✗'})" - ) + image_broken = self._has_broken_image(series) + status = f"imdb_id={'✓' if has_imdb else '✗'}" + if image_broken: + status += ", image=BROKEN" + elif has_image: + status += ", image=✓" + else: + status += ", image=✗" + self.stdout.write(f" [DRY RUN] Would fix {series.name} ({status})") return updated = 0 errors = 0 - for series in qs.iterator(): + for series in all_series: try: with transaction.atomic(): - series.fix_metadata(force_update=force) + series.fix_metadata(force_update=force or self._has_broken_image(series)) updated += 1 self.stdout.write(f" [{updated}/{total}] {series.name}") except Exception as e: