[webpages] Add images to webpages

vrobbler/apps/webpages/migrations/0007_webpage_image.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.2.29 on 2026-03-14 17:50
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("webpages", "0006_remove_webpage_run_time_seconds_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="webpage",
+            name="image",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="webpage/"
+            ),
+        ),
+    ]

vrobbler/apps/webpages/models.py

@@ -1,19 +1,23 @@
 import logging
-from typing import Dict
+from typing import Dict, Optional
 from uuid import uuid4
-from django_extensions.db.models import TimeStampedModel
 
 import pendulum
 import requests
-from taggit.managers import TaggableManager
 import trafilatura
+from bs4 import BeautifulSoup
 from django.apps import apps
 from django.conf import settings
 from django.contrib.auth import get_user_model
+from django.core.files.base import ContentFile
 from django.db import models
 from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
 from htmldate import find_date
+from imagekit.models import ImageSpecField
+from imagekit.processors import ResizeToFit
 from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
+from taggit.managers import TaggableManager
 
 logger = logging.getLogger(__name__)
 BNULL = {"blank": True, "null": True}
@@ -35,9 +39,9 @@ class Domain(TimeStampedModel):
     def scrobbles_for_user(self, user_id):
         from scrobbles.models import Scrobble
 
-        return Scrobble.objects.filter(
+        return Scrobble.objects.filter(web_page__domain=self, user_id=user_id).order_by(
-            web_page__domain=self, user_id=user_id
+            "-timestamp"
-        ).order_by("-timestamp")
+        )
 
 
 class WebPage(ScrobblableMixin):
@@ -49,6 +53,20 @@ class WebPage(ScrobblableMixin):
     domain = models.ForeignKey(Domain, on_delete=models.DO_NOTHING, **BNULL)
     extract = models.TextField(**BNULL)
 
+    image = models.ImageField(upload_to="webpage/", **BNULL)
+    image_small = ImageSpecField(
+        source="image",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    image_medium = ImageSpecField(
+        source="image",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
+
     def __str__(self) -> str:
         if self.title:
             return "{} ({})".format(self.title, self.domain)
@@ -95,7 +113,8 @@ class WebPage(ScrobblableMixin):
 
     @property
     def primary_image_url(self) -> str:
-        # TODO Figure out how to add a preview?
+        if self.image:
+            return self.image.url
         return ""
 
     def scrobble_for_user(self, user_id):
@@ -117,9 +136,7 @@ class WebPage(ScrobblableMixin):
 
     def scrobbles(self, user):
         Scrobble = apps.get_model("scrobbles", "Scrobble")
-        return Scrobble.objects.filter(user=user, web_page=self).order_by(
+        return Scrobble.objects.filter(user=user, web_page=self).order_by("-timestamp")
-            "-timestamp"
-        )
 
     def clean_title(self, title: str, save=True):
         if len(title.split("|")) > 1:
@@ -144,9 +161,7 @@ class WebPage(ScrobblableMixin):
         if not raw_text:
             return
 
-        self.title = raw_text[
+        self.title = raw_text[raw_text.find("<title>") + 7 : raw_text.find("</title>")]
-            raw_text.find("<title>") + 7 : raw_text.find("</title>")
-        ]
 
         if not self.title and self.extract:
             first_line = self.extract.split("\n")[0]
@@ -194,14 +209,47 @@ class WebPage(ScrobblableMixin):
             return
 
         if response.status_code == 200:
-            logger.info(
+            logger.info("Website already exists in archive", extra={"url": self.url})
-                "Website already exists in archive", extra={"url": self.url}
-            )
         else:
             raise Exception(
                 f"Failed to push URL to archivebox (Response {response.status_code})"
             )
 
+    def _extract_image_url_from_html(self, raw_html: str) -> Optional[str]:
+        """Extract og:image or twitter:image from HTML."""
+        if not raw_html:
+            return None
+
+        soup = BeautifulSoup(raw_html, "html.parser")
+
+        og_image = soup.find("meta", property="og:image")
+        if og_image and og_image.get("content"):
+            return og_image["content"]
+
+        twitter_image = soup.find("meta", attrs={"name": "twitter:image"})
+        if twitter_image and twitter_image.get("content"):
+            return twitter_image["content"]
+
+        return None
+
+    def _download_and_save_image(self, image_url: str) -> bool:
+        """Download and save the webpage image."""
+        if not image_url:
+            return False
+
+        try:
+            response = requests.get(image_url, timeout=10)
+            response.raise_for_status()
+
+            filename = image_url.split("/")[-1].split("?")[0]
+            if not filename.endswith((".jpg", ".jpeg", ".png", ".webp")):
+                filename = "webpage_image.jpg"
+
+            self.image.save(filename, ContentFile(response.content), save=True)
+            return True
+        except Exception:
+            return False
+
     def fetch_data_from_web(self, save=True, force=True):
         raw_text = trafilatura.fetch_url(self.url)
         if not self.extract or force:
@@ -223,6 +271,11 @@ class WebPage(ScrobblableMixin):
         if not self.base_run_time_seconds or force:
             self.base_run_time_seconds = self.estimated_time_to_read_in_seconds
 
+        if (not self.image or force) and raw_text:
+            image_url = self._extract_image_url_from_html(raw_text)
+            if image_url:
+                self._download_and_save_image(image_url)
+
         if save:
             self.save()