[webpages] Add autotagging to webpages

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [0/20] :vrobbler:project:personal:
+* Backlog [1/21] :vrobbler:project:personal:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles:
 :PROPERTIES:
 :ID:       702462cf-d54b-48c6-8a7c-78b8de751deb
@@ -590,6 +590,18 @@ We should rename `email_scrobble_board_game` to reflect the fact that it's just
 a helper method to create board game scrobbles given a json blob. It's
 independent of the email flow it was originally creatdd for
 
+** DONE [#B] Add autotagging to webpages based on domain, title :webpages:metadata:
+:PROPERTIES:
+:ID:       f658435b-f7a0-42e6-b9f6-226678a77a55
+:END:
+
+*** Description
+
+For easier filtering, like we do with tasks, we should auto tag WebPage instances
+based on the domain name split part by periods (so news.ycombinator.com tags: news, ycombinator, com)
+
+And also based on the nouns in the title.
+
 * Version 54.5 [1/1]
 ** DONE Fix bug in generating mood trends :trends:
 :PROPERTIES:

vrobbler/apps/scrobbles/utils.py

@@ -1,4 +1,5 @@
 import hashlib
+import html
 import logging
 import requests
 import re
@@ -795,6 +796,7 @@ def tokenize_title_to_tags(title: str) -> list[str]:
     if not title:
         return []
 
+    title = html.unescape(title)
     cleaned = re.sub(r"[\(\)\[\]\{\}]", "", title)
     cleaned = re.sub(r"[^\w\s]", "", cleaned)
 

vrobbler/apps/webpages/apps.py

@@ -3,3 +3,6 @@ from django.apps import AppConfig
 
 class WebpagesConfig(AppConfig):
     name = "webpages"
+
+    def ready(self):
+        import webpages.signals  # noqa

vrobbler/apps/webpages/management/commands/backfill_webpage_tags.py

@@ -0,0 +1,68 @@
+import re
+
+from django.core.management.base import BaseCommand
+
+from scrobbles.utils import tokenize_title_to_tags
+from webpages.models import WebPage
+
+
+def _clean(s: str) -> str:
+    return re.sub(r"[^\x20-\x7e]", "", s)
+
+
+class Command(BaseCommand):
+    help = "Backfill auto tags on webpages from domain and title"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--commit",
+            action="store_true",
+            help="Actually add tags",
+        )
+
+    def handle(self, *args, **options):
+        commit = options["commit"]
+
+        webpages = WebPage.objects.all()
+        total = webpages.count()
+        updated_count = 0
+        skipped_count = 0
+
+        for i, webpage in enumerate(webpages.iterator(), start=1):
+            new_tags = set()
+
+            if webpage.domain:
+                parts = webpage.domain.root.split(".")
+                for part in parts:
+                    part = part.strip().lower()
+                    if part and part != "www":
+                        new_tags.add(part)
+
+            if webpage.title:
+                title_tags = tokenize_title_to_tags(webpage.title)
+                new_tags.update(title_tags)
+
+            existing_tags = {
+                t.name for t in webpage.tags.all()
+            }
+            tags_to_add = new_tags - existing_tags
+
+            if tags_to_add:
+                updated_count += 1
+                if commit:
+                    for tag in tags_to_add:
+                        webpage.tags.add(tag)
+                    self.stdout.write(
+                        f"[{i}/{total}] Added tags to {_clean(str(webpage))}: "
+                        f"{sorted(tags_to_add)}"
+                    )
+                else:
+                    self.stdout.write(
+                        f"[{i}/{total}] [DRY RUN] Would add tags to "
+                        f"{_clean(str(webpage))}: {sorted(tags_to_add)}"
+                    )
+            else:
+                skipped_count += 1
+
+        self.stdout.write(f"\nDone. {updated_count} webpages to update, "
+                          f"{skipped_count} already up to date.")

vrobbler/apps/webpages/signals.py

@@ -0,0 +1,29 @@
+import logging
+
+from django.db.models.signals import post_save
+from django.dispatch import receiver
+
+from scrobbles.utils import tokenize_title_to_tags
+from webpages.models import WebPage
+
+logger = logging.getLogger(__name__)
+
+
+@receiver(post_save, sender=WebPage)
+def add_auto_tags_to_webpage(sender, instance, **kwargs):
+    existing_tags = {t.name for t in instance.tags.all()}
+
+    if instance.domain:
+        domain_parts = instance.domain.root.split(".")
+        for part in domain_parts:
+            part = part.strip().lower()
+            if part and part != "www" and part not in existing_tags:
+                instance.tags.add(part)
+                existing_tags.add(part)
+
+    if instance.title:
+        title_tags = tokenize_title_to_tags(instance.title)
+        for tag in title_tags:
+            if tag not in existing_tags:
+                instance.tags.add(tag)
+                existing_tags.add(tag)

vrobbler/apps/webpages/views.py

@@ -16,8 +16,23 @@ from webpages.models import WebPage
 class WebPageListView(ScrobbleableListView):
     model = WebPage
 
+    def get_queryset(self):
+        queryset = super().get_queryset()
+        tag_name = self.request.GET.get("tag")
+        if tag_name:
+            from django.contrib.contenttypes.models import ContentType
+            from taggit.models import TaggedItem
+            ct = ContentType.objects.get_for_model(WebPage)
+            webpages = TaggedItem.objects.filter(
+                content_type=ct,
+                tag__name__iexact=tag_name,
+            ).values_list("object_id", flat=True)
+            queryset = queryset.filter(pk__in=webpages)
+        return queryset
+
     def get_context_data(self, **kwargs):
         context_data = super().get_context_data(**kwargs)
+        context_data["active_tag"] = self.request.GET.get("tag", "")
         user = self.request.user
         now = timezone.now()
         start_day_of_week = now - datetime.timedelta(days=now.weekday())

vrobbler/templates/webpages/webpage_detail.html

@@ -204,6 +204,13 @@
       <p>Source: <a href="{{object.url}}">{{object.domain}}</a></p>
       {% if object.date %}<p>Published: <em>{{object.date}}</em></p>{% endif %}
       <p>Time to read: {{object.estimated_time_to_read_in_minutes}} minutes</p>
+      {% if object.tags.all %}
+      <p>Tags:
+        {% for tag in object.tags.all %}
+        <a href="{% url 'webpages:webpage_list' %}?tag={{ tag.name|urlencode }}">{{ tag.name }}</a>{% if not forloop.last %}, {% endif %}
+        {% endfor %}
+      </p>
+      {% endif %}
     </div>
     {% if object.extract %}
     <div class="col">