From 4e1db4aa6fc393878b0c15012b35f154d7f5fb87 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Thu, 30 Nov 2023 17:49:53 +0100 Subject: [PATCH] Add webpage scrobbling --- ...obble_webpage_alter_scrobble_media_type.py | 44 +++++++++++ vrobbler/apps/scrobbles/models.py | 21 +++-- vrobbler/apps/scrobbles/scrobblers.py | 23 +++++- vrobbler/apps/scrobbles/views.py | 3 + vrobbler/apps/webpages/__init__.py | 0 vrobbler/apps/webpages/apps.py | 5 ++ .../apps/webpages/migrations/0001_initial.py | 79 +++++++++++++++++++ vrobbler/apps/webpages/migrations/__init__.py | 0 vrobbler/apps/webpages/models.py | 65 +++++++++++++++ vrobbler/settings-testing.py | 1 + vrobbler/settings.py | 1 + 11 files changed, 231 insertions(+), 11 deletions(-) create mode 100644 vrobbler/apps/scrobbles/migrations/0045_scrobble_webpage_alter_scrobble_media_type.py create mode 100644 vrobbler/apps/webpages/__init__.py create mode 100644 vrobbler/apps/webpages/apps.py create mode 100644 vrobbler/apps/webpages/migrations/0001_initial.py create mode 100644 vrobbler/apps/webpages/migrations/__init__.py create mode 100644 vrobbler/apps/webpages/models.py diff --git a/vrobbler/apps/scrobbles/migrations/0045_scrobble_webpage_alter_scrobble_media_type.py b/vrobbler/apps/scrobbles/migrations/0045_scrobble_webpage_alter_scrobble_media_type.py new file mode 100644 index 0000000..c5d3dca --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0045_scrobble_webpage_alter_scrobble_media_type.py @@ -0,0 +1,44 @@ +# Generated by Django 4.1.7 on 2023-11-30 16:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("webpages", "0001_initial"), + ("scrobbles", "0044_scrobble_geo_location_alter_scrobble_media_type"), + ] + + operations = [ + migrations.AddField( + model_name="scrobble", + name="webpage", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to="webpages.webpage", + ), + ), + migrations.AlterField( + model_name="scrobble", + name="media_type", + field=models.CharField( + choices=[ + ("Video", "Video"), + ("Track", "Track"), + ("Episode", "Podcast episode"), + ("SportEvent", "Sport event"), + ("Book", "Book"), + ("VideoGame", "Video game"), + ("BoardGame", "Board game"), + ("GeoLocation", "GeoLocation"), + ("WebPage", "Web Page"), + ], + default="Video", + max_length=14, + ), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 855399c..6b745d5 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -14,6 +14,7 @@ from django.utils import timezone from django.utils.functional import cached_property from django_extensions.db.models import TimeStampedModel from locations.models import GeoLocation +from webpages.models import WebPage from music.lastfm import LastFM from music.models import Artist, Track from podcasts.models import Episode @@ -471,6 +472,7 @@ class Scrobble(TimeStampedModel): VIDEO_GAME = "VideoGame", "Video game" BOARD_GAME = "BoardGame", "Board game" GEO_LOCATION = "GeoLocation", "GeoLocation" + WEBPAGE = "WebPage", "Web Page" uuid = models.UUIDField(editable=False, **BNULL) video = models.ForeignKey(Video, on_delete=models.DO_NOTHING, **BNULL) @@ -491,6 +493,7 @@ class Scrobble(TimeStampedModel): geo_location = models.ForeignKey( GeoLocation, on_delete=models.DO_NOTHING, **BNULL ) + webpage = models.ForeignKey(WebPage, on_delete=models.DO_NOTHING, **BNULL) media_type = models.CharField( max_length=14, choices=MediaType.choices, default=MediaType.VIDEO ) @@ -697,18 +700,23 @@ class Scrobble(TimeStampedModel): if media_class == "BoardGame": media_query = models.Q(board_game=media) scrobble_data["board_game_id"] = media.id + if media_class == "WebPage": + media_query = models.Q(webpage=media) + scrobble_data["webpage_id"] = media.id if media_class == "GeoLocation": media_query = models.Q(media_type=Scrobble.MediaType.GEO_LOCATION) scrobble_data["geo_location_id"] = media.id dup = cls.objects.filter( - media_type=cls.MediaType.GEO_LOCATION, - timestamp = scrobble_data.get("timestamp"), + media_type=cls.MediaType.GEO_LOCATION, + timestamp=scrobble_data.get("timestamp"), ).first() - if dup: - logger.info("[scrobbling] scrobble with identical timestamp found") - return - + if dup: + logger.info( + "[scrobbling] scrobble for geo location with identical timestamp found" + ) + # TODO Fix return type, can we ever return a Scrobble? + return scrobble = ( cls.objects.filter( @@ -719,7 +727,6 @@ class Scrobble(TimeStampedModel): .first() ) - if scrobble and scrobble.can_be_updated: source = scrobble_data["source"] mtype = media.__class__.__name__ diff --git a/vrobbler/apps/scrobbles/scrobblers.py b/vrobbler/apps/scrobbles/scrobblers.py index 81515d6..c472134 100644 --- a/vrobbler/apps/scrobbles/scrobblers.py +++ b/vrobbler/apps/scrobbles/scrobblers.py @@ -1,13 +1,15 @@ import logging -import pendulum from typing import Optional +import pendulum from boardgames.bgg import lookup_boardgame_from_bgg from boardgames.models import BoardGame from books.models import Book from books.openlibrary import lookup_book_from_openlibrary from dateutil.parser import parse from django.utils import timezone +from locations.constants import LOCATION_PROVIDERS +from locations.models import GeoLocation from music.constants import JELLYFIN_POST_KEYS from music.models import Track from music.utils import ( @@ -23,8 +25,7 @@ from sports.thesportsdb import lookup_event_from_thesportsdb from videogames.howlongtobeat import lookup_game_from_hltb from videogames.models import VideoGame from videos.models import Video -from locations.models import GeoLocation, RawGeoLocation -from locations.constants import LOCATION_PROVIDERS +from webpages.models import WebPage logger = logging.getLogger(__name__) @@ -247,6 +248,20 @@ def manual_scrobble_board_game(bggeek_id: str, user_id: int): return Scrobble.create_or_update(boardgame, user_id, scrobble_dict) +def manual_scrobble_webpage(url: str, user_id: int): + webpage = WebPage.find_or_create({"url": url}) + + scrobble_dict = { + "user_id": user_id, + "timestamp": timezone.now(), + "playback_position_seconds": 0, + "source": "Vrobbler", + "source_id": "Manually scrobbled from Vrobbler", + } + + return Scrobble.create_or_update(webpage, user_id, scrobble_dict) + + def gpslogger_scrobble_location( data_dict: dict, user_id: Optional[int] ) -> Optional[Scrobble]: @@ -268,7 +283,7 @@ def gpslogger_scrobble_location( scrobble = Scrobble.create_or_update(location, user_id, extra_data) provider = f"data source: {LOCATION_PROVIDERS[data_dict.get('prov')]}" - if scrobble: + if scrobble: if scrobble.notes: scrobble.notes = scrobble.notes + f"\n{provider}" else: diff --git a/vrobbler/apps/scrobbles/views.py b/vrobbler/apps/scrobbles/views.py index 83aa286..6d7004f 100644 --- a/vrobbler/apps/scrobbles/views.py +++ b/vrobbler/apps/scrobbles/views.py @@ -65,6 +65,7 @@ from scrobbles.utils import ( get_long_plays_in_progress, get_recently_played_board_games, ) +from vrobbler.apps.scrobbles.scrobblers import manual_scrobble_webpage logger = logging.getLogger(__name__) @@ -441,6 +442,8 @@ def scrobble_start(request, uuid): scrobble = manual_scrobble_video_game(media_obj.hltb_id, user_id) if media_obj.__class__.__name__ == Scrobble.MediaType.BOARD_GAME: scrobble = manual_scrobble_board_game(media_obj.bggeek_id, user_id) + if media_obj.__class__.__name__ == Scrobble.MediaType.WEBPAGE: + scrobble = manual_scrobble_webpage(media_obj.url, user_id) if scrobble: messages.add_message( diff --git a/vrobbler/apps/webpages/__init__.py b/vrobbler/apps/webpages/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/webpages/apps.py b/vrobbler/apps/webpages/apps.py new file mode 100644 index 0000000..f488c81 --- /dev/null +++ b/vrobbler/apps/webpages/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class WebpagesConfig(AppConfig): + name = "webpages" diff --git a/vrobbler/apps/webpages/migrations/0001_initial.py b/vrobbler/apps/webpages/migrations/0001_initial.py new file mode 100644 index 0000000..580b9a5 --- /dev/null +++ b/vrobbler/apps/webpages/migrations/0001_initial.py @@ -0,0 +1,79 @@ +# Generated by Django 4.1.7 on 2023-11-30 16:45 + +from django.db import migrations, models +import django_extensions.db.fields +import taggit.managers +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("scrobbles", "0044_scrobble_geo_location_alter_scrobble_media_type"), + ] + + operations = [ + migrations.CreateModel( + name="WebPage", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name="created" + ), + ), + ( + "modified", + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name="modified" + ), + ), + ( + "title", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "run_time_seconds", + models.IntegerField(blank=True, null=True), + ), + ( + "run_time_ticks", + models.PositiveBigIntegerField(blank=True, null=True), + ), + ( + "uuid", + models.UUIDField( + blank=True, + default=uuid.uuid4, + editable=False, + null=True, + ), + ), + ("url", models.URLField(max_length=500)), + ( + "genre", + taggit.managers.TaggableManager( + blank=True, + help_text="A comma-separated list of tags.", + through="scrobbles.ObjectWithGenres", + to="scrobbles.Genre", + verbose_name="Tags", + ), + ), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/vrobbler/apps/webpages/migrations/__init__.py b/vrobbler/apps/webpages/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/webpages/models.py b/vrobbler/apps/webpages/models.py new file mode 100644 index 0000000..7fc1162 --- /dev/null +++ b/vrobbler/apps/webpages/models.py @@ -0,0 +1,65 @@ +import requests +import logging +from typing import Dict +from uuid import uuid4 + +from django.contrib.auth import get_user_model +from django.conf import settings +from django.db import models +from django.urls import reverse +from scrobbles.mixins import ScrobblableMixin + +logger = logging.getLogger(__name__) +BNULL = {"blank": True, "null": True} +User = get_user_model() + + +class WebPage(ScrobblableMixin): + COMPLETION_PERCENT = getattr(settings, "WEBSITE_COMPLETION_PERCENT", 100) + + uuid = models.UUIDField(default=uuid4, editable=False, **BNULL) + url = models.URLField(max_length=500) + + def __str__(self): + if self.title: + return self.title + + return self.domain + + @property + def domain(self): + self.url.split("//")[-1].split("/")[0] + + def get_absolute_url(self): + return reverse( + "locations:geo_location_detail", kwargs={"slug": self.uuid} + ) + + def _update_title_from_web(self, force=False): + headers = { + "headers": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:51.0) Gecko/20100101 Firefox/51.0" + } + raw_text = requests.get(self.url, headers=headers).text + if not self.title or force: + self.title = raw_text[ + raw_text.find("") + 7 : raw_text.find("") + ] + self.save(update_fields=["title"]) + + @classmethod + def find_or_create(cls, data_dict: Dict) -> "GeoLocation": + """Given a data dict from an manual URL scrobble, does the heavy lifting of looking up + the url, creating if if doesn't exist yet. + + """ + # TODO Add constants for all these data keys + if "url" not in data_dict.keys(): + logger.error("No url in data dict") + return + + webpage = cls.objects.filter(url=data_dict.get("url")).first() + + if not webpage: + webpage = cls.objects.create(url=data_dict.get("url")) + webpage._update_title_from_web() + return webpage diff --git a/vrobbler/settings-testing.py b/vrobbler/settings-testing.py index 59b4306..0699426 100644 --- a/vrobbler/settings-testing.py +++ b/vrobbler/settings-testing.py @@ -107,6 +107,7 @@ INSTALLED_APPS = [ "boardgames", "videogames", "locations", + "webpages", "mathfilters", "rest_framework", "allauth", diff --git a/vrobbler/settings.py b/vrobbler/settings.py index ee8d05b..2f067bf 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -115,6 +115,7 @@ INSTALLED_APPS = [ "boardgames", "videogames", "locations", + "webpages", "mathfilters", "rest_framework", "allauth",