diff --git a/PROJECT.org b/PROJECT.org index 95989ef..be48a39 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [1/21] :vrobbler:project:personal: +* Backlog [2/22] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -594,6 +594,26 @@ named constants for maintainability. - ~vrobbler/apps/webpages/models.py~ (line 290) -- ="url"= - ~vrobbler/apps/scrobbles/importers/tsv.py~ (line 55) -- ="S"= completion status +** DONE [#B] Webpage scrobbles should diff existing webpages content :webpages:metadata: +:PROPERTIES: +:ID: 25576197-258f-48d6-bfe9-e4172a0a1898 +:END: + +*** Description + +Webpages change content between scrobbles. The current model stores the webpage content once, the +first time it's scrobbled. When a page has been seen before, we should move the existing content +to a new model HistoricalWebPage with the following fields: + +webpage_id -> FK to WebPage +date -> date from existing WebPage content +domain -> same as existing WebPage content +extract -> copy of existing WebPage content + +Once the HistoricalWebPage instance is successfully created, the new extract data +should be saved into the WebPage instance. + + ** DONE [#B] Make ArchiveBox push asynchronous :archivebox:async: :PROPERTIES: :ID: 17c116a7-5952-db37-e56c-2987c2fc456b diff --git a/vrobbler/apps/webpages/admin.py b/vrobbler/apps/webpages/admin.py index 5664f1d..059b7fb 100644 --- a/vrobbler/apps/webpages/admin.py +++ b/vrobbler/apps/webpages/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin -from webpages.models import Domain, WebPage +from webpages.models import Domain, HistoricalWebPage, WebPage from scrobbles.admin import ScrobbleInline @@ -20,6 +20,12 @@ class DomainAdmin(admin.ModelAdmin): inlines = [WebPageInline] +class HistoricalWebPageInline(admin.TabularInline): + model = HistoricalWebPage + extra = 0 + readonly_fields = ("date", "domain", "extract", "created") + + @admin.register(WebPage) class WebPageAdmin(admin.ModelAdmin): date_hierarchy = "created" @@ -33,4 +39,20 @@ class WebPageAdmin(admin.ModelAdmin): search_fields = ("title",) inlines = [ ScrobbleInline, + HistoricalWebPageInline, ] + + +@admin.register(HistoricalWebPage) +class HistoricalWebPageAdmin(admin.ModelAdmin): + date_hierarchy = "created" + list_display = ( + "uuid", + "webpage", + "date", + "domain", + "created", + ) + raw_id_fields = ("webpage", "domain") + ordering = ("-created",) + search_fields = ("webpage__title",) diff --git a/vrobbler/apps/webpages/migrations/0010_historicalwebpage.py b/vrobbler/apps/webpages/migrations/0010_historicalwebpage.py new file mode 100644 index 0000000..7f89507 --- /dev/null +++ b/vrobbler/apps/webpages/migrations/0010_historicalwebpage.py @@ -0,0 +1,71 @@ +# Generated by Django 4.2.29 on 2026-06-16 13:39 + +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ("webpages", "0009_alter_webpage_genre"), + ] + + operations = [ + migrations.CreateModel( + name="HistoricalWebPage", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "uuid", + models.UUIDField( + blank=True, default=uuid.uuid4, editable=False, null=True + ), + ), + ("date", models.DateField(blank=True, null=True)), + ("extract", models.TextField(blank=True, null=True)), + ( + "domain", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="webpages.domain", + ), + ), + ( + "webpage", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="historical_webpages", + to="webpages.webpage", + ), + ), + ], + options={ + "get_latest_by": "modified", + "abstract": False, + }, + ), + ] diff --git a/vrobbler/apps/webpages/models.py b/vrobbler/apps/webpages/models.py index 3b6a8e8..9da6593 100644 --- a/vrobbler/apps/webpages/models.py +++ b/vrobbler/apps/webpages/models.py @@ -303,4 +303,41 @@ class WebPage(ScrobblableMixin): if not webpage: webpage = cls(url=data_dict.get("url")) webpage.fetch_data_from_web(save=True) + else: + webpage._archive_and_refetch() return webpage + + def _archive_and_refetch(self): + """Archive current content to HistoricalWebPage and re-fetch from web.""" + if self.extract or self.date or self.domain: + HistoricalWebPage.objects.create( + webpage=self, + date=self.date, + domain=self.domain, + extract=self.extract, + ) + + self.extract = None + self.date = None + self.domain = None + self.title = None + self.base_run_time_seconds = None + self.image = None + self.fetch_data_from_web(save=True, force=True) + + +class HistoricalWebPage(TimeStampedModel): + uuid = models.UUIDField(default=uuid4, editable=False, **BNULL) + webpage = models.ForeignKey( + WebPage, on_delete=models.CASCADE, related_name="historical_webpages" + ) + date = models.DateField(**BNULL) + domain = models.ForeignKey(Domain, on_delete=models.DO_NOTHING, **BNULL) + extract = models.TextField(**BNULL) + + def __str__(self) -> str: + if self.webpage.title: + return "{} ({}) - {}".format( + self.webpage.title, self.webpage.domain, self.created + ) + return "{} - {}".format(self.webpage.url, self.created)