Add webpage scrobbling

vrobbler/apps/scrobbles/migrations/0045_scrobble_webpage_alter_scrobble_media_type.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.1.7 on 2023-11-30 16:45
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("webpages", "0001_initial"),
+        ("scrobbles", "0044_scrobble_geo_location_alter_scrobble_media_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scrobble",
+            name="webpage",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="webpages.webpage",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="scrobble",
+            name="media_type",
+            field=models.CharField(
+                choices=[
+                    ("Video", "Video"),
+                    ("Track", "Track"),
+                    ("Episode", "Podcast episode"),
+                    ("SportEvent", "Sport event"),
+                    ("Book", "Book"),
+                    ("VideoGame", "Video game"),
+                    ("BoardGame", "Board game"),
+                    ("GeoLocation", "GeoLocation"),
+                    ("WebPage", "Web Page"),
+                ],
+                default="Video",
+                max_length=14,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -14,6 +14,7 @@ from django.utils import timezone
 from django.utils.functional import cached_property
 from django_extensions.db.models import TimeStampedModel
 from locations.models import GeoLocation
+from webpages.models import WebPage
 from music.lastfm import LastFM
 from music.models import Artist, Track
 from podcasts.models import Episode
@@ -471,6 +472,7 @@ class Scrobble(TimeStampedModel):
         VIDEO_GAME = "VideoGame", "Video game"
         BOARD_GAME = "BoardGame", "Board game"
         GEO_LOCATION = "GeoLocation", "GeoLocation"
+        WEBPAGE = "WebPage", "Web Page"
 
     uuid = models.UUIDField(editable=False, **BNULL)
     video = models.ForeignKey(Video, on_delete=models.DO_NOTHING, **BNULL)
@@ -491,6 +493,7 @@ class Scrobble(TimeStampedModel):
     geo_location = models.ForeignKey(
         GeoLocation, on_delete=models.DO_NOTHING, **BNULL
     )
+    webpage = models.ForeignKey(WebPage, on_delete=models.DO_NOTHING, **BNULL)
     media_type = models.CharField(
         max_length=14, choices=MediaType.choices, default=MediaType.VIDEO
     )
@@ -697,18 +700,23 @@ class Scrobble(TimeStampedModel):
         if media_class == "BoardGame":
             media_query = models.Q(board_game=media)
             scrobble_data["board_game_id"] = media.id
+        if media_class == "WebPage":
+            media_query = models.Q(webpage=media)
+            scrobble_data["webpage_id"] = media.id
         if media_class == "GeoLocation":
             media_query = models.Q(media_type=Scrobble.MediaType.GEO_LOCATION)
             scrobble_data["geo_location_id"] = media.id
             dup = cls.objects.filter(
-                    media_type=cls.MediaType.GEO_LOCATION,
-                    timestamp = scrobble_data.get("timestamp"),
+                media_type=cls.MediaType.GEO_LOCATION,
+                timestamp=scrobble_data.get("timestamp"),
             ).first()
 
-        if dup:
-            logger.info("[scrobbling] scrobble with identical timestamp found")
-            return
-
+            if dup:
+                logger.info(
+                    "[scrobbling] scrobble for geo location with identical timestamp found"
+                )
+                # TODO Fix return type, can we ever return a Scrobble?
+                return
 
         scrobble = (
             cls.objects.filter(
@@ -719,7 +727,6 @@ class Scrobble(TimeStampedModel):
             .first()
         )
 
-
         if scrobble and scrobble.can_be_updated:
             source = scrobble_data["source"]
             mtype = media.__class__.__name__

vrobbler/apps/scrobbles/scrobblers.py

@@ -1,13 +1,15 @@
 import logging
-import pendulum
 from typing import Optional
 
+import pendulum
 from boardgames.bgg import lookup_boardgame_from_bgg
 from boardgames.models import BoardGame
 from books.models import Book
 from books.openlibrary import lookup_book_from_openlibrary
 from dateutil.parser import parse
 from django.utils import timezone
+from locations.constants import LOCATION_PROVIDERS
+from locations.models import GeoLocation
 from music.constants import JELLYFIN_POST_KEYS
 from music.models import Track
 from music.utils import (
@@ -23,8 +25,7 @@ from sports.thesportsdb import lookup_event_from_thesportsdb
 from videogames.howlongtobeat import lookup_game_from_hltb
 from videogames.models import VideoGame
 from videos.models import Video
-from locations.models import GeoLocation, RawGeoLocation
-from locations.constants import LOCATION_PROVIDERS
+from webpages.models import WebPage
 
 logger = logging.getLogger(__name__)
 
@@ -247,6 +248,20 @@ def manual_scrobble_board_game(bggeek_id: str, user_id: int):
     return Scrobble.create_or_update(boardgame, user_id, scrobble_dict)
 
 
+def manual_scrobble_webpage(url: str, user_id: int):
+    webpage = WebPage.find_or_create({"url": url})
+
+    scrobble_dict = {
+        "user_id": user_id,
+        "timestamp": timezone.now(),
+        "playback_position_seconds": 0,
+        "source": "Vrobbler",
+        "source_id": "Manually scrobbled from Vrobbler",
+    }
+
+    return Scrobble.create_or_update(webpage, user_id, scrobble_dict)
+
+
 def gpslogger_scrobble_location(
     data_dict: dict, user_id: Optional[int]
 ) -> Optional[Scrobble]:
@@ -268,7 +283,7 @@ def gpslogger_scrobble_location(
     scrobble = Scrobble.create_or_update(location, user_id, extra_data)
 
     provider = f"data source: {LOCATION_PROVIDERS[data_dict.get('prov')]}"
-    if scrobble: 
+    if scrobble:
         if scrobble.notes:
             scrobble.notes = scrobble.notes + f"\n{provider}"
         else:

vrobbler/apps/scrobbles/views.py

@@ -65,6 +65,7 @@ from scrobbles.utils import (
     get_long_plays_in_progress,
     get_recently_played_board_games,
 )
+from vrobbler.apps.scrobbles.scrobblers import manual_scrobble_webpage
 
 logger = logging.getLogger(__name__)
 
@@ -441,6 +442,8 @@ def scrobble_start(request, uuid):
             scrobble = manual_scrobble_video_game(media_obj.hltb_id, user_id)
         if media_obj.__class__.__name__ == Scrobble.MediaType.BOARD_GAME:
             scrobble = manual_scrobble_board_game(media_obj.bggeek_id, user_id)
+        if media_obj.__class__.__name__ == Scrobble.MediaType.WEBPAGE:
+            scrobble = manual_scrobble_webpage(media_obj.url, user_id)
 
     if scrobble:
         messages.add_message(

vrobbler/apps/webpages/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class WebpagesConfig(AppConfig):
+    name = "webpages"

vrobbler/apps/webpages/migrations/0001_initial.py

@@ -0,0 +1,79 @@
+# Generated by Django 4.1.7 on 2023-11-30 16:45
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import taggit.managers
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("scrobbles", "0044_scrobble_geo_location_alter_scrobble_media_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="WebPage",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "title",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "run_time_seconds",
+                    models.IntegerField(blank=True, null=True),
+                ),
+                (
+                    "run_time_ticks",
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ("url", models.URLField(max_length=500)),
+                (
+                    "genre",
+                    taggit.managers.TaggableManager(
+                        blank=True,
+                        help_text="A comma-separated list of tags.",
+                        through="scrobbles.ObjectWithGenres",
+                        to="scrobbles.Genre",
+                        verbose_name="Tags",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

vrobbler/apps/webpages/models.py

@@ -0,0 +1,65 @@
+import requests
+import logging
+from typing import Dict
+from uuid import uuid4
+
+from django.contrib.auth import get_user_model
+from django.conf import settings
+from django.db import models
+from django.urls import reverse
+from scrobbles.mixins import ScrobblableMixin
+
+logger = logging.getLogger(__name__)
+BNULL = {"blank": True, "null": True}
+User = get_user_model()
+
+
+class WebPage(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, "WEBSITE_COMPLETION_PERCENT", 100)
+
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    url = models.URLField(max_length=500)
+
+    def __str__(self):
+        if self.title:
+            return self.title
+
+        return self.domain
+
+    @property
+    def domain(self):
+        self.url.split("//")[-1].split("/")[0]
+
+    def get_absolute_url(self):
+        return reverse(
+            "locations:geo_location_detail", kwargs={"slug": self.uuid}
+        )
+
+    def _update_title_from_web(self, force=False):
+        headers = {
+            "headers": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:51.0) Gecko/20100101 Firefox/51.0"
+        }
+        raw_text = requests.get(self.url, headers=headers).text
+        if not self.title or force:
+            self.title = raw_text[
+                raw_text.find("<title>") + 7 : raw_text.find("</title>")
+            ]
+            self.save(update_fields=["title"])
+
+    @classmethod
+    def find_or_create(cls, data_dict: Dict) -> "GeoLocation":
+        """Given a data dict from an manual URL scrobble, does the heavy lifting of looking up
+        the url, creating if if doesn't exist yet.
+
+        """
+        # TODO Add constants for all these data keys
+        if "url" not in data_dict.keys():
+            logger.error("No url in data dict")
+            return
+
+        webpage = cls.objects.filter(url=data_dict.get("url")).first()
+
+        if not webpage:
+            webpage = cls.objects.create(url=data_dict.get("url"))
+            webpage._update_title_from_web()
+        return webpage

vrobbler/settings-testing.py

@@ -107,6 +107,7 @@ INSTALLED_APPS = [
     "boardgames",
     "videogames",
     "locations",
+    "webpages",
     "mathfilters",
     "rest_framework",
     "allauth",

vrobbler/settings.py

@@ -115,6 +115,7 @@ INSTALLED_APPS = [
     "boardgames",
     "videogames",
     "locations",
+    "webpages",
     "mathfilters",
     "rest_framework",
     "allauth",