diff --git a/PROJECT.org b/PROJECT.org index fada774..2cd272a 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -88,7 +88,7 @@ fetching and simple saving. *** Metadata sources **** Scraper -* Backlog [0/20] :vrobbler:project:personal: +* Backlog [1/21] :vrobbler:project:personal: ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles: :PROPERTIES: :ID: 702462cf-d54b-48c6-8a7c-78b8de751deb @@ -590,6 +590,18 @@ We should rename `email_scrobble_board_game` to reflect the fact that it's just a helper method to create board game scrobbles given a json blob. It's independent of the email flow it was originally creatdd for +** DONE [#B] Add autotagging to webpages based on domain, title :webpages:metadata: +:PROPERTIES: +:ID: f658435b-f7a0-42e6-b9f6-226678a77a55 +:END: + +*** Description + +For easier filtering, like we do with tasks, we should auto tag WebPage instances +based on the domain name split part by periods (so news.ycombinator.com tags: news, ycombinator, com) + +And also based on the nouns in the title. + * Version 54.5 [1/1] ** DONE Fix bug in generating mood trends :trends: :PROPERTIES: diff --git a/vrobbler/apps/scrobbles/utils.py b/vrobbler/apps/scrobbles/utils.py index 21b3384..be5346f 100644 --- a/vrobbler/apps/scrobbles/utils.py +++ b/vrobbler/apps/scrobbles/utils.py @@ -1,4 +1,5 @@ import hashlib +import html import logging import requests import re @@ -795,6 +796,7 @@ def tokenize_title_to_tags(title: str) -> list[str]: if not title: return [] + title = html.unescape(title) cleaned = re.sub(r"[\(\)\[\]\{\}]", "", title) cleaned = re.sub(r"[^\w\s]", "", cleaned) diff --git a/vrobbler/apps/webpages/apps.py b/vrobbler/apps/webpages/apps.py index f488c81..a0964c6 100644 --- a/vrobbler/apps/webpages/apps.py +++ b/vrobbler/apps/webpages/apps.py @@ -3,3 +3,6 @@ from django.apps import AppConfig class WebpagesConfig(AppConfig): name = "webpages" + + def ready(self): + import webpages.signals # noqa diff --git a/vrobbler/apps/webpages/management/__init__.py b/vrobbler/apps/webpages/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/webpages/management/commands/__init__.py b/vrobbler/apps/webpages/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/webpages/management/commands/backfill_webpage_tags.py b/vrobbler/apps/webpages/management/commands/backfill_webpage_tags.py new file mode 100644 index 0000000..9df487e --- /dev/null +++ b/vrobbler/apps/webpages/management/commands/backfill_webpage_tags.py @@ -0,0 +1,68 @@ +import re + +from django.core.management.base import BaseCommand + +from scrobbles.utils import tokenize_title_to_tags +from webpages.models import WebPage + + +def _clean(s: str) -> str: + return re.sub(r"[^\x20-\x7e]", "", s) + + +class Command(BaseCommand): + help = "Backfill auto tags on webpages from domain and title" + + def add_arguments(self, parser): + parser.add_argument( + "--commit", + action="store_true", + help="Actually add tags", + ) + + def handle(self, *args, **options): + commit = options["commit"] + + webpages = WebPage.objects.all() + total = webpages.count() + updated_count = 0 + skipped_count = 0 + + for i, webpage in enumerate(webpages.iterator(), start=1): + new_tags = set() + + if webpage.domain: + parts = webpage.domain.root.split(".") + for part in parts: + part = part.strip().lower() + if part and part != "www": + new_tags.add(part) + + if webpage.title: + title_tags = tokenize_title_to_tags(webpage.title) + new_tags.update(title_tags) + + existing_tags = { + t.name for t in webpage.tags.all() + } + tags_to_add = new_tags - existing_tags + + if tags_to_add: + updated_count += 1 + if commit: + for tag in tags_to_add: + webpage.tags.add(tag) + self.stdout.write( + f"[{i}/{total}] Added tags to {_clean(str(webpage))}: " + f"{sorted(tags_to_add)}" + ) + else: + self.stdout.write( + f"[{i}/{total}] [DRY RUN] Would add tags to " + f"{_clean(str(webpage))}: {sorted(tags_to_add)}" + ) + else: + skipped_count += 1 + + self.stdout.write(f"\nDone. {updated_count} webpages to update, " + f"{skipped_count} already up to date.") diff --git a/vrobbler/apps/webpages/signals.py b/vrobbler/apps/webpages/signals.py new file mode 100644 index 0000000..2a34a7b --- /dev/null +++ b/vrobbler/apps/webpages/signals.py @@ -0,0 +1,29 @@ +import logging + +from django.db.models.signals import post_save +from django.dispatch import receiver + +from scrobbles.utils import tokenize_title_to_tags +from webpages.models import WebPage + +logger = logging.getLogger(__name__) + + +@receiver(post_save, sender=WebPage) +def add_auto_tags_to_webpage(sender, instance, **kwargs): + existing_tags = {t.name for t in instance.tags.all()} + + if instance.domain: + domain_parts = instance.domain.root.split(".") + for part in domain_parts: + part = part.strip().lower() + if part and part != "www" and part not in existing_tags: + instance.tags.add(part) + existing_tags.add(part) + + if instance.title: + title_tags = tokenize_title_to_tags(instance.title) + for tag in title_tags: + if tag not in existing_tags: + instance.tags.add(tag) + existing_tags.add(tag) diff --git a/vrobbler/apps/webpages/views.py b/vrobbler/apps/webpages/views.py index ce1df06..8b6aa1e 100644 --- a/vrobbler/apps/webpages/views.py +++ b/vrobbler/apps/webpages/views.py @@ -16,8 +16,23 @@ from webpages.models import WebPage class WebPageListView(ScrobbleableListView): model = WebPage + def get_queryset(self): + queryset = super().get_queryset() + tag_name = self.request.GET.get("tag") + if tag_name: + from django.contrib.contenttypes.models import ContentType + from taggit.models import TaggedItem + ct = ContentType.objects.get_for_model(WebPage) + webpages = TaggedItem.objects.filter( + content_type=ct, + tag__name__iexact=tag_name, + ).values_list("object_id", flat=True) + queryset = queryset.filter(pk__in=webpages) + return queryset + def get_context_data(self, **kwargs): context_data = super().get_context_data(**kwargs) + context_data["active_tag"] = self.request.GET.get("tag", "") user = self.request.user now = timezone.now() start_day_of_week = now - datetime.timedelta(days=now.weekday()) diff --git a/vrobbler/templates/webpages/webpage_detail.html b/vrobbler/templates/webpages/webpage_detail.html index 6307d67..01fc85b 100644 --- a/vrobbler/templates/webpages/webpage_detail.html +++ b/vrobbler/templates/webpages/webpage_detail.html @@ -204,6 +204,13 @@
Source: {{object.domain}}
{% if object.date %}Published: {{object.date}}
{% endif %}Time to read: {{object.estimated_time_to_read_in_minutes}} minutes
+ {% if object.tags.all %} +Tags: + {% for tag in object.tags.all %} + {{ tag.name }}{% if not forloop.last %}, {% endif %} + {% endfor %} +
+ {% endif %} {% if object.extract %}