[webpages] Add historical extract stashing

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [1/21] :vrobbler:project:personal:
+* Backlog [2/22] :vrobbler:project:personal:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles:
 :PROPERTIES:
 :ID:       702462cf-d54b-48c6-8a7c-78b8de751deb
@@ -594,6 +594,26 @@ named constants for maintainability.
   - ~vrobbler/apps/webpages/models.py~ (line 290) -- ="url"=
   - ~vrobbler/apps/scrobbles/importers/tsv.py~ (line 55) -- ="S"= completion status
 
+** DONE [#B] Webpage scrobbles should diff existing webpages content :webpages:metadata:
+:PROPERTIES:
+:ID:       25576197-258f-48d6-bfe9-e4172a0a1898
+:END:
+
+*** Description
+
+Webpages change content between scrobbles. The current model stores the webpage content once, the
+first time it's scrobbled. When a page has been seen before, we should move the existing content
+to a new model HistoricalWebPage with the following fields:
+
+webpage_id -> FK to WebPage
+date -> date from existing WebPage content
+domain -> same as existing WebPage content
+extract -> copy of existing WebPage content
+
+Once the HistoricalWebPage instance is successfully created, the new extract data
+should be saved into the WebPage instance.
+
+
 ** DONE [#B] Make ArchiveBox push asynchronous :archivebox:async:
 :PROPERTIES:
 :ID:       17c116a7-5952-db37-e56c-2987c2fc456b

vrobbler/apps/webpages/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-from webpages.models import Domain, WebPage
+from webpages.models import Domain, HistoricalWebPage, WebPage
 
 from scrobbles.admin import ScrobbleInline
 
@@ -20,6 +20,12 @@ class DomainAdmin(admin.ModelAdmin):
     inlines = [WebPageInline]
 
 
+class HistoricalWebPageInline(admin.TabularInline):
+    model = HistoricalWebPage
+    extra = 0
+    readonly_fields = ("date", "domain", "extract", "created")
+
+
 @admin.register(WebPage)
 class WebPageAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
@@ -33,4 +39,20 @@ class WebPageAdmin(admin.ModelAdmin):
     search_fields = ("title",)
     inlines = [
         ScrobbleInline,
+        HistoricalWebPageInline,
     ]
+
+
+@admin.register(HistoricalWebPage)
+class HistoricalWebPageAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "uuid",
+        "webpage",
+        "date",
+        "domain",
+        "created",
+    )
+    raw_id_fields = ("webpage", "domain")
+    ordering = ("-created",)
+    search_fields = ("webpage__title",)

vrobbler/apps/webpages/migrations/0010_historicalwebpage.py

@@ -0,0 +1,71 @@
+# Generated by Django 4.2.29 on 2026-06-16 13:39
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("webpages", "0009_alter_webpage_genre"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="HistoricalWebPage",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True, default=uuid.uuid4, editable=False, null=True
+                    ),
+                ),
+                ("date", models.DateField(blank=True, null=True)),
+                ("extract", models.TextField(blank=True, null=True)),
+                (
+                    "domain",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to="webpages.domain",
+                    ),
+                ),
+                (
+                    "webpage",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="historical_webpages",
+                        to="webpages.webpage",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

vrobbler/apps/webpages/models.py

@@ -303,4 +303,41 @@ class WebPage(ScrobblableMixin):
         if not webpage:
             webpage = cls(url=data_dict.get("url"))
             webpage.fetch_data_from_web(save=True)
+        else:
+            webpage._archive_and_refetch()
         return webpage
+
+    def _archive_and_refetch(self):
+        """Archive current content to HistoricalWebPage and re-fetch from web."""
+        if self.extract or self.date or self.domain:
+            HistoricalWebPage.objects.create(
+                webpage=self,
+                date=self.date,
+                domain=self.domain,
+                extract=self.extract,
+            )
+
+        self.extract = None
+        self.date = None
+        self.domain = None
+        self.title = None
+        self.base_run_time_seconds = None
+        self.image = None
+        self.fetch_data_from_web(save=True, force=True)
+
+
+class HistoricalWebPage(TimeStampedModel):
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    webpage = models.ForeignKey(
+        WebPage, on_delete=models.CASCADE, related_name="historical_webpages"
+    )
+    date = models.DateField(**BNULL)
+    domain = models.ForeignKey(Domain, on_delete=models.DO_NOTHING, **BNULL)
+    extract = models.TextField(**BNULL)
+
+    def __str__(self) -> str:
+        if self.webpage.title:
+            return "{} ({}) - {}".format(
+                self.webpage.title, self.webpage.domain, self.created
+            )
+        return "{} - {}".format(self.webpage.url, self.created)