diff --git a/vrobbler/apps/webpages/migrations/0007_webpage_image.py b/vrobbler/apps/webpages/migrations/0007_webpage_image.py new file mode 100644 index 0000000..4c8e394 --- /dev/null +++ b/vrobbler/apps/webpages/migrations/0007_webpage_image.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.29 on 2026-03-14 17:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("webpages", "0006_remove_webpage_run_time_seconds_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="webpage", + name="image", + field=models.ImageField( + blank=True, null=True, upload_to="webpage/" + ), + ), + ] diff --git a/vrobbler/apps/webpages/models.py b/vrobbler/apps/webpages/models.py index c15ff98..5442779 100644 --- a/vrobbler/apps/webpages/models.py +++ b/vrobbler/apps/webpages/models.py @@ -1,19 +1,23 @@ import logging -from typing import Dict +from typing import Dict, Optional from uuid import uuid4 -from django_extensions.db.models import TimeStampedModel import pendulum import requests -from taggit.managers import TaggableManager import trafilatura +from bs4 import BeautifulSoup from django.apps import apps from django.conf import settings from django.contrib.auth import get_user_model +from django.core.files.base import ContentFile from django.db import models from django.urls import reverse +from django_extensions.db.models import TimeStampedModel from htmldate import find_date +from imagekit.models import ImageSpecField +from imagekit.processors import ResizeToFit from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin +from taggit.managers import TaggableManager logger = logging.getLogger(__name__) BNULL = {"blank": True, "null": True} @@ -35,9 +39,9 @@ class Domain(TimeStampedModel): def scrobbles_for_user(self, user_id): from scrobbles.models import Scrobble - return Scrobble.objects.filter( - web_page__domain=self, user_id=user_id - ).order_by("-timestamp") + return Scrobble.objects.filter(web_page__domain=self, user_id=user_id).order_by( + "-timestamp" + ) class WebPage(ScrobblableMixin): @@ -49,6 +53,20 @@ class WebPage(ScrobblableMixin): domain = models.ForeignKey(Domain, on_delete=models.DO_NOTHING, **BNULL) extract = models.TextField(**BNULL) + image = models.ImageField(upload_to="webpage/", **BNULL) + image_small = ImageSpecField( + source="image", + processors=[ResizeToFit(100, 100)], + format="JPEG", + options={"quality": 60}, + ) + image_medium = ImageSpecField( + source="image", + processors=[ResizeToFit(300, 300)], + format="JPEG", + options={"quality": 75}, + ) + def __str__(self) -> str: if self.title: return "{} ({})".format(self.title, self.domain) @@ -95,7 +113,8 @@ class WebPage(ScrobblableMixin): @property def primary_image_url(self) -> str: - # TODO Figure out how to add a preview? + if self.image: + return self.image.url return "" def scrobble_for_user(self, user_id): @@ -117,9 +136,7 @@ class WebPage(ScrobblableMixin): def scrobbles(self, user): Scrobble = apps.get_model("scrobbles", "Scrobble") - return Scrobble.objects.filter(user=user, web_page=self).order_by( - "-timestamp" - ) + return Scrobble.objects.filter(user=user, web_page=self).order_by("-timestamp") def clean_title(self, title: str, save=True): if len(title.split("|")) > 1: @@ -144,9 +161,7 @@ class WebPage(ScrobblableMixin): if not raw_text: return - self.title = raw_text[ - raw_text.find("") + 7 : raw_text.find("") - ] + self.title = raw_text[raw_text.find("") + 7 : raw_text.find("")] if not self.title and self.extract: first_line = self.extract.split("\n")[0] @@ -194,14 +209,47 @@ class WebPage(ScrobblableMixin): return if response.status_code == 200: - logger.info( - "Website already exists in archive", extra={"url": self.url} - ) + logger.info("Website already exists in archive", extra={"url": self.url}) else: raise Exception( f"Failed to push URL to archivebox (Response {response.status_code})" ) + def _extract_image_url_from_html(self, raw_html: str) -> Optional[str]: + """Extract og:image or twitter:image from HTML.""" + if not raw_html: + return None + + soup = BeautifulSoup(raw_html, "html.parser") + + og_image = soup.find("meta", property="og:image") + if og_image and og_image.get("content"): + return og_image["content"] + + twitter_image = soup.find("meta", attrs={"name": "twitter:image"}) + if twitter_image and twitter_image.get("content"): + return twitter_image["content"] + + return None + + def _download_and_save_image(self, image_url: str) -> bool: + """Download and save the webpage image.""" + if not image_url: + return False + + try: + response = requests.get(image_url, timeout=10) + response.raise_for_status() + + filename = image_url.split("/")[-1].split("?")[0] + if not filename.endswith((".jpg", ".jpeg", ".png", ".webp")): + filename = "webpage_image.jpg" + + self.image.save(filename, ContentFile(response.content), save=True) + return True + except Exception: + return False + def fetch_data_from_web(self, save=True, force=True): raw_text = trafilatura.fetch_url(self.url) if not self.extract or force: @@ -223,6 +271,11 @@ class WebPage(ScrobblableMixin): if not self.base_run_time_seconds or force: self.base_run_time_seconds = self.estimated_time_to_read_in_seconds + if (not self.image or force) and raw_text: + image_url = self._extract_image_url_from_html(raw_text) + if image_url: + self._download_and_save_image(image_url) + if save: self.save()