Remove whitespace on podcast names and episode titles

The way we calculate past seconds for long plays should be much less
error prone and uses our built-in previous scrobble accessor for media types.

.drone.yml

@@ -4,17 +4,16 @@
 ################
 
 kind: pipeline
-name: run_tests
+name: build & deploy
 
 steps:
   # Run tests against Python/Flask engine backend (with pytest)
   - name: pytest with coverage
-    image: python:3.10.4
+    image: python:3.11.1
     commands:
       # Install dependencies
       - cp vrobbler.conf.test vrobbler.conf
       - pip install poetry
-      - poetry config repositories.unblink "https://pypi.unbl.ink/pypi/simple"
       - poetry install
       # Start with a fresh database (which is already running as a service from Drone)
       - poetry run pytest --cov-report term:skip-covered --cov=vrobbler tests
@@ -42,6 +41,30 @@ steps:
     when:
       branch:
       - main
+  - name: build success notification
+    image: parrazam/drone-ntfy
+    when:
+        status: [success]
+    settings:
+      url: https://ntfy.unbl.ink
+      topic: drone
+      priority: low
+      tags:
+        - cd
+        - failure
+        - vrobbler
+  - name: build failure notification
+    image: parrazam/drone-ntfy
+    when:
+        status: [failure]
+    settings:
+      url: https://ntfy.unbl.ink
+      topic: drone
+      priority: high
+      tags:
+        - cd
+        - success
+        - vrobbler
 volumes:
   - name: docker
     host:

README.md

@@ -1,7 +1,7 @@
 Vrobbler
 ========
 
-[![Build Status](https://ci.unbl.ink/api/badges/secstate/vrobbler/status.svg?ref=refs/heads/main)](https://ci.unbl.ink/secstate/vrobbler)
+[![Build Status](https://ci.lab.unbl.ink/api/badges/secstate/vrobbler/status.svg?ref=refs/heads/main)](https://ci.lab.unbl.ink/secstate/vrobbler)
 
 Vrobbler is a pretty simple Django-powered web app for scrobbling video plays from you favorite Jellyfin installation.
 

pyproject.toml

@@ -5,7 +5,7 @@ description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = ">=3.9,<4.0"
 Django = "^4.0.3"
 django-extensions = "^3.1.5"
 python-dateutil = "^2.8.2"
@@ -40,6 +40,10 @@ beautifulsoup4 = "^4.11.2"
 django-storages = "^1.13.2"
 boto3 = "^1.26.98"
 stream-sqlite = "^0.0.41"
+ipython = "^8.14.0"
+pendulum = "^2.1.2"
+trafilatura = "^1.6.3"
+django-imagekit = "^5.0.0"
 
 [tool.poetry.dev-dependencies]
 Werkzeug = "2.0.3"
@@ -63,7 +67,7 @@ bandit = "^1.7.4"
 minversion = "6.0"
 addopts = "-ra -q"
 testpaths = ["tests"]
-DJANGO_SETTINGS_MODULE='vrobbler.settings'
+DJANGO_SETTINGS_MODULE='vrobbler.settings-testing'
 
 [tool.black]
 line-length = 79

tests/boardgames_tests/test_bgg.py

@@ -0,0 +1,27 @@
+from boardgames.bgg import (
+    take_first,
+    lookup_boardgame_id_from_bgg,
+    lookup_boardgame_from_bgg,
+)
+
+
+def test_take_first():
+    assert take_first([]) == ""
+
+    assert take_first(["a", "b"]) == "a"
+
+
+def test_lookup_boardgame_id_from_bgg():
+    bgg_id = lookup_boardgame_id_from_bgg("Cosmic Encounter")
+    assert bgg_id == "15"
+
+    bgg_id = lookup_boardgame_id_from_bgg("Comedy Encounter")
+    assert bgg_id == None
+
+
+def test_lookup_boardgame_from_bgg():
+    bgg_result = lookup_boardgame_from_bgg(15)
+    assert bgg_result.get("bggeek_id") == 15
+
+    bgg_result = lookup_boardgame_from_bgg("Cosmic Encounter")
+    assert bgg_result.get("bggeek_id") == "15"

tests/podcasts_tests/test_scrapers.py

@@ -1,19 +1,10 @@
 from vrobbler.apps.podcasts.scrapers import scrape_data_from_google_podcasts
 
-expected_desc = (
+expected_desc_snippet = (
     "NPR's Up First is the news you need to start your day. "
-    "The three biggest stories of the day, with reporting and analysis "
-    "from NPR News — in 10 minutes. Available weekdays by 6 a.m. ET, "
-    "with hosts Leila Fadel, Steve Inskeep, Michel Martin and A Martinez. "
-    "Also available on Saturdays by 8 a.m. ET, with Ayesha Rascoe and "
-    "Scott Simon. On Sundays, hear a longer exploration behind the "
-    "headlines with Rachel Martin, available by 8 a.m. ET. Subscribe "
-    "and listen, then support your local NPR station at donate.npr.org.  "
-    "Support NPR's reporting by subscribing to Up First+ and unlock "
-    "sponsor-free listening. Learn more at plus.npr.org/UpFirst"
 )
 
-expected_img_url = "https://encrypted-tbn2.gstatic.com/images?q=tbn:ANd9GcT-PqtK-bauo8wm8dBE__SVGArlvfBYY8rqxr2kA5UwjKzEx8c"
+expected_img_url = "https://encrypted-tbn2.gstatic.com/images?q=tbn:ANd9GcR1F0CfR24RR6sme531yIkCrnK4zzmo97jeualO5drVPKG6oCk"
 expected_google_url = "https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzUxMDMxOC9wb2RjYXN0LnhtbA"
 
 
@@ -22,7 +13,7 @@ def test_get_not_allowed_from_mopidy():
     result_dict = scrape_data_from_google_podcasts(query)
 
     assert result_dict["title"] == query
-    assert result_dict["description"] == expected_desc
+    assert expected_desc_snippet in result_dict["description"]
     assert result_dict["image_url"] == expected_img_url
     assert result_dict["producer"] == "NPR"
     assert result_dict["google_url"] == expected_google_url

tests/scrobbles_tests/test_aggregators.py

@@ -1,4 +1,6 @@
 from datetime import datetime, timedelta
+from unittest import mock
+from unittest.mock import patch
 
 import pytest
 import time_machine
@@ -6,6 +8,7 @@ from django.contrib.auth import get_user_model
 from django.urls import reverse
 from django.utils import timezone
 from music.aggregators import live_charts, scrobble_counts, week_of_scrobbles
+from music.models import Album, Artist
 from profiles.models import UserProfile
 from scrobbles.models import Scrobble
 
@@ -39,64 +42,31 @@ def test_scrobble_counts_data(client, mopidy_track_request_data):
 
 
 @pytest.mark.django_db
-def test_week_of_scrobbles_data(client, mopidy_track_request_data):
+@time_machine.travel(datetime(2022, 3, 4, 1, 24))
+def test_live_charts(client, mopidy_track_request_data):
     build_scrobbles(client, mopidy_track_request_data, 7, 1)
     user = get_user_model().objects.first()
+
     week = week_of_scrobbles(user)
     assert list(week.values()) == [1, 1, 1, 1, 1, 1, 1]
 
-
-@pytest.mark.django_db
-def test_top_tracks_by_day(client, mopidy_track_request_data):
-    build_scrobbles(client, mopidy_track_request_data, 7, 1)
-    user = get_user_model().objects.first()
     tops = live_charts(user)
     assert tops[0].title == "Same in the End"
 
-
-@pytest.mark.django_db
-def test_top_tracks_by_week(client, mopidy_track_request_data):
-    build_scrobbles(client, mopidy_track_request_data, 7, 1)
-    user = get_user_model().objects.first()
     tops = live_charts(user, chart_period="week")
     assert tops[0].title == "Same in the End"
 
-
-@pytest.mark.django_db
-def test_top_tracks_by_month(client, mopidy_track_request_data):
-    build_scrobbles(client, mopidy_track_request_data, 7, 1)
-    user = get_user_model().objects.first()
     tops = live_charts(user, chart_period="month")
     assert tops[0].title == "Same in the End"
 
-
-@pytest.mark.django_db
-def test_top_tracks_by_year(client, mopidy_track_request_data):
-    build_scrobbles(client, mopidy_track_request_data, 7, 1)
-    user = get_user_model().objects.first()
     tops = live_charts(user, chart_period="year")
     assert tops[0].title == "Same in the End"
 
-
-@pytest.mark.django_db
-def test_top__artists_by_week(client, mopidy_track_request_data):
-    build_scrobbles(client, mopidy_track_request_data, 7, 1)
-    user = get_user_model().objects.first()
     tops = live_charts(user, chart_period="week", media_type="Artist")
     assert tops[0].name == "Sublime"
 
-
-@pytest.mark.django_db
-def test_top__artists_by_month(client, mopidy_track_request_data):
-    build_scrobbles(client, mopidy_track_request_data, 7, 1)
-    user = get_user_model().objects.first()
     tops = live_charts(user, chart_period="month", media_type="Artist")
     assert tops[0].name == "Sublime"
 
-
-@pytest.mark.django_db
-def test_top__artists_by_year(client, mopidy_track_request_data):
-    build_scrobbles(client, mopidy_track_request_data, 7, 1)
-    user = get_user_model().objects.first()
     tops = live_charts(user, chart_period="year", media_type="Artist")
     assert tops[0].name == "Sublime"

tests/scrobbles_tests/test_utils.py

@@ -0,0 +1,12 @@
+from datetime import datetime
+import pytz
+from django.contrib.auth import get_user_model
+
+from vrobbler.apps.scrobbles.utils import timestamp_user_tz_to_utc
+
+
+def test_timestamp_user_tz_to_utc():
+    timestamp = timestamp_user_tz_to_utc(
+        1685561082, pytz.timezone("US/Eastern")
+    )
+    assert timestamp == datetime(2023, 5, 31, 23, 24, 42, tzinfo=pytz.utc)

todos.org

@@ -3,6 +3,10 @@
 A fun way to keep track of things in the project to fix or improve.
 
 * Version 1.0.0
+** TODO What to do with Youtube videos from LastFM and web-scrobbler :bug:source:lastfm:
+** TODO Add a user profile page with ability to change settings :profiles:improvement:
+** DONE [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :utililty:improvement:
+CLOSED: [2023-04-06 Thu 14:09]
 ** DONE Add a "stop_timestamp" so we don't rely on content length :improvement:scrobbling:
 CLOSED: [2023-04-02 Sun 23:58]
 
@@ -149,6 +153,7 @@ Of course, it could also be a data model without a table, where it runs some fun
 calculations, stores it's values in Redis as a long-term lookup table and just
 has to re-populate when the server restarts.
 * Backlog
+** TODO Add Amazon scraper to look up books when OL fails :books:improvement:
 ** TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :utility:improvement:
 *** Example payloads from mopidy-webhooks
 **** Podcast playback ended
@@ -440,7 +445,5 @@ has to re-populate when the server restarts.
     }
 }
 #+end_src
-** TODO [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :utililty:improvement:
-** TODO What to do with Youtube videos from LastFM and web-scrobbler :bug:source:lastfm:
 ** TODO Fix bug in Jellyfin scrobbles that spam more scrobbles after completion :scrobbling:videos:bug:
 ** TODO Fix bug in podcast scrobbling where a second scrobble is created after completion :scrobbling:podcasts:bug:

vrobbler.conf.test

@@ -1,8 +1,11 @@
 # Local configuration for Emus
 
-VROBBLER_DUMP_REQUEST_DATA=True
-VROBBLER_LOG_TO_CONSOLE=True
-VROBBLER_DEBUG=True
+VROBBLER_DUMP_REQUEST_DATA=False
+VROBBLER_LOG_TO_CONSOLE=False
+VROBBLER_DEBUG=False
 VROBBLER_LOG_LEVEL="DEBUG"
 VROBBLER_MEDIA_ROOT = "/tmp/media/"
-VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True
+VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=False
+
+VROBBLER_USE_S3="False"
+VROBBLER_DATABASE_URL="sqlite:///testdb.sqlite3"

vrobbler/apps/boardgames/admin.py

@@ -0,0 +1,30 @@
+from django.contrib import admin
+
+from boardgames.models import BoardGame, BoardGamePublisher
+
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(BoardGamePublisher)
+class BoardGamePublisherAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "name",
+        "uuid",
+    )
+    ordering = ("-created",)
+
+
+@admin.register(BoardGame)
+class GameAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "bggeek_id",
+        "title",
+        "published_date",
+    )
+    search_fields = ("title",)
+    ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/boardgames/bgg.py

@@ -0,0 +1,92 @@
+import logging
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+SEARCH_ID_URL = (
+    "https://boardgamegeek.com/xmlapi/search?search={query}&exact=1"
+)
+GAME_ID_URL = "https://boardgamegeek.com/xmlapi/boardgame/{id}"
+
+
+def take_first(thing: Optional[list]) -> str:
+    first = ""
+    try:
+        first = thing[0]
+    except IndexError:
+        pass
+
+    if first:
+        try:
+            first = first.get_text()
+        except:
+            pass
+
+    return first
+
+
+def lookup_boardgame_id_from_bgg(title: str) -> Optional[int]:
+    soup = None
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    game_id = None
+    url = SEARCH_ID_URL.format(query=title)
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "xml")
+
+    if soup:
+        result = soup.findAll("boardgame")
+        if not result:
+            return game_id
+
+        game_id = result[0].get("objectid", None)
+
+    return game_id
+
+
+def lookup_boardgame_from_bgg(lookup_id: str) -> dict:
+    soup = None
+    game_dict = {}
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+
+    title = ""
+    bgg_id = None
+
+    try:
+        bgg_id = int(lookup_id)
+        logger.debug(f"Using BGG ID {bgg_id} to find board game")
+    except ValueError:
+        title = lookup_id
+        logger.debug(f"Using title {title} to find board game")
+
+    if not bgg_id:
+        bgg_id = lookup_boardgame_id_from_bgg(title)
+
+    url = GAME_ID_URL.format(id=bgg_id)
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "xml")
+
+    if soup:
+        seconds_to_play = None
+        minutes = take_first(soup.findAll("playingtime"))
+        if minutes:
+            seconds_to_play = int(minutes) * 60
+
+        game_dict = {
+            "bggeek_id": bgg_id,
+            "title": take_first(soup.findAll("name", primary="true")),
+            "description": take_first(soup.findAll("description")),
+            "year_published": take_first(soup.findAll("yearpublished")),
+            "publisher_name": take_first(soup.findAll("boardgamepublisher")),
+            "cover_url": take_first(soup.findAll("image")),
+            "min_players": take_first(soup.findAll("minplayers")),
+            "max_players": take_first(soup.findAll("maxplayers")),
+            "recommended_age": take_first(soup.findAll("age")),
+            "run_time_seconds": seconds_to_play,
+        }
+
+    return game_dict

vrobbler/apps/boardgames/migrations/0001_initial.py

@@ -0,0 +1,168 @@
+# Generated by Django 4.1.7 on 2023-04-17 22:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import taggit.managers
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("scrobbles", "0038_alter_objectwithgenres_tag"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="BoardGamePublisher",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("name", models.CharField(max_length=255)),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    "logo",
+                    models.ImageField(
+                        blank=True,
+                        null=True,
+                        upload_to="games/platform-logos/",
+                    ),
+                ),
+                ("igdb_id", models.IntegerField(blank=True, null=True)),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="BoardGame",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "run_time_seconds",
+                    models.IntegerField(blank=True, null=True),
+                ),
+                (
+                    "run_time_ticks",
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                ("title", models.CharField(max_length=255)),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    "cover",
+                    models.ImageField(
+                        blank=True, null=True, upload_to="boardgames/covers/"
+                    ),
+                ),
+                (
+                    "layout_image",
+                    models.ImageField(
+                        blank=True, null=True, upload_to="boardgames/layouts/"
+                    ),
+                ),
+                ("summary", models.TextField(blank=True, null=True)),
+                ("rating", models.FloatField(blank=True, null=True)),
+                (
+                    "max_players",
+                    models.PositiveSmallIntegerField(blank=True, null=True),
+                ),
+                (
+                    "min_players",
+                    models.PositiveSmallIntegerField(blank=True, null=True),
+                ),
+                ("published_date", models.DateField(blank=True, null=True)),
+                (
+                    "recommened_age",
+                    models.PositiveSmallIntegerField(blank=True, null=True),
+                ),
+                (
+                    "seconds_to_play",
+                    models.IntegerField(blank=True, null=True),
+                ),
+                (
+                    "bggeek_id",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "genre",
+                    taggit.managers.TaggableManager(
+                        help_text="A comma-separated list of tags.",
+                        through="scrobbles.ObjectWithGenres",
+                        to="scrobbles.Genre",
+                        verbose_name="Tags",
+                    ),
+                ),
+                (
+                    "publisher",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to="boardgames.boardgamepublisher",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0002_boardgame_description.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-04-17 22:16
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="boardgame",
+            name="description",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0003_rename_recommened_age_boardgame_recommended_age.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-04-17 22:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0002_boardgame_description"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="boardgame",
+            old_name="recommened_age",
+            new_name="recommended_age",
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0004_remove_boardgame_seconds_to_play.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.7 on 2023-04-17 22:25
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0003_rename_recommened_age_boardgame_recommended_age"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="boardgame",
+            name="seconds_to_play",
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0005_remove_boardgame_summary.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.7 on 2023-04-17 22:29
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0004_remove_boardgame_seconds_to_play"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="boardgame",
+            name="summary",
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0006_alter_boardgame_genre.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-04-18 02:33
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+        ("boardgames", "0005_remove_boardgame_summary"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="boardgame",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                blank=True,
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/boardgames/models.py

@@ -0,0 +1,166 @@
+import logging
+from datetime import datetime
+from typing import Optional
+from uuid import uuid4
+
+import requests
+from boardgames.bgg import lookup_boardgame_from_bgg
+from django.conf import settings
+from django.core.files.base import ContentFile
+from django.db import models
+from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
+from imagekit.models import ImageSpecField
+from imagekit.processors import ResizeToFit
+from scrobbles.mixins import ScrobblableMixin
+from vrobbler.apps.boardgames.bgg import lookup_boardgame_id_from_bgg
+
+logger = logging.getLogger(__name__)
+BNULL = {"blank": True, "null": True}
+
+
+class BoardGamePublisher(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    logo = models.ImageField(upload_to="games/platform-logos/", **BNULL)
+    igdb_id = models.IntegerField(**BNULL)
+
+    def __str__(self):
+        return self.name
+
+    def get_absolute_url(self):
+        return reverse(
+            "boardgames:publisher_detail", kwargs={"slug": self.uuid}
+        )
+
+
+class BoardGame(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(
+        settings, "BOARD_GAME_COMPLETION_PERCENT", 100
+    )
+
+    FIELDS_FROM_BGGEEK = [
+        "igdb_id",
+        "alternative_name",
+        "rating",
+        "rating_count",
+        "release_date",
+        "cover",
+        "screenshot",
+    ]
+
+    title = models.CharField(max_length=255)
+    publisher = models.ForeignKey(
+        BoardGamePublisher, **BNULL, on_delete=models.DO_NOTHING
+    )
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    description = models.TextField(**BNULL)
+    cover = models.ImageField(upload_to="boardgames/covers/", **BNULL)
+    cover_small = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    cover_medium = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
+    layout_image = models.ImageField(upload_to="boardgames/layouts/", **BNULL)
+    layout_image_small = ImageSpecField(
+        source="layout_image",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    layout_image_medium = ImageSpecField(
+        source="layout_image",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
+    rating = models.FloatField(**BNULL)
+    max_players = models.PositiveSmallIntegerField(**BNULL)
+    min_players = models.PositiveSmallIntegerField(**BNULL)
+    published_date = models.DateField(**BNULL)
+    recommended_age = models.PositiveSmallIntegerField(**BNULL)
+    bggeek_id = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return self.title
+
+    def get_absolute_url(self):
+        return reverse(
+            "boardgames:boardgame_detail", kwargs={"slug": self.uuid}
+        )
+
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.cover:
+            url = self.cover.url
+        return url
+
+    def bggeek_link(self):
+        link = ""
+        if self.bggeek_id:
+            link = f"https://boardgamegeek.com/boardgame/{self.bggeek_id}"
+        return link
+
+    def get_start_url(self):
+        return reverse("scrobbles:start", kwargs={"uuid": self.uuid})
+
+    def fix_metadata(self, data: dict = {}, force_update=False) -> None:
+
+        if not self.published_date or force_update:
+
+            if not data:
+                data = lookup_boardgame_from_bgg(str(self.bggeek_id))
+
+            cover_url = data.pop("cover_url")
+            year = data.pop("year_published")
+            publisher_name = data.pop("publisher_name")
+
+            if year:
+                data["published_date"] = datetime(int(year), 1, 1)
+
+            # Fun trick for updating all fields at once
+            BoardGame.objects.filter(pk=self.id).update(**data)
+            self.refresh_from_db()
+
+            # Add publishers
+            (
+                self.publisher,
+                _created,
+            ) = BoardGamePublisher.objects.get_or_create(name=publisher_name)
+            self.save()
+
+            # Go get cover image if the URL is present
+            if cover_url and not self.cover:
+                headers = {"User-Agent": "Vrobbler 0.11.12"}
+                r = requests.get(cover_url, headers=headers)
+                logger.debug(r.status_code)
+                if r.status_code == 200:
+                    fname = f"{self.title}_cover_{self.uuid}.jpg"
+                    self.cover.save(fname, ContentFile(r.content), save=True)
+                    logger.debug("Loaded cover image from BGGeek")
+
+    @classmethod
+    def find_or_create(
+        cls, lookup_id: str, data: Optional[dict] = {}
+    ) -> Optional["BoardGame"]:
+        """Given a Lookup ID (either BGG or BGA ID), return a board game object"""
+        boardgame = None
+
+        if not data:
+            data = lookup_boardgame_from_bgg(lookup_id)
+
+        if data:
+            boardgame, created = cls.objects.get_or_create(
+                title=data["title"], bggeek_id=lookup_id
+            )
+            if created:
+                boardgame.fix_metadata(data=data)
+
+        return boardgame

vrobbler/apps/boardgames/urls.py

@@ -0,0 +1,21 @@
+from django.urls import path
+from boardgames import views
+
+app_name = "boardgames"
+
+
+urlpatterns = [
+    path(
+        "board-game/", views.BoardGameListView.as_view(), name="boardgame_list"
+    ),
+    path(
+        "board-game/<slug:slug>/",
+        views.BoardGameDetailView.as_view(),
+        name="boardgame_detail",
+    ),
+    path(
+        "board-game-publisher/<slug:slug>/",
+        views.BoardGamePublisherDetailView.as_view(),
+        name="publisher_detail",
+    ),
+]

vrobbler/apps/boardgames/views.py

@@ -0,0 +1,17 @@
+from django.views import generic
+from boardgames.models import BoardGame, BoardGamePublisher
+
+
+class BoardGameListView(generic.ListView):
+    model = BoardGame
+    paginate_by = 20
+
+
+class BoardGameDetailView(generic.DetailView):
+    model = BoardGame
+    slug_field = "uuid"
+
+
+class BoardGamePublisherDetailView(generic.DetailView):
+    model = BoardGamePublisher
+    slug_field = "uuid"

vrobbler/apps/books/amazon.py

@@ -0,0 +1,144 @@
+from enum import Enum
+from typing import Optional
+from bs4 import BeautifulSoup
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+USER_AGENT = (
+    "Mozilla/5.0 (Android 4.4; Mobile; rv:41.0) Gecko/41.0 Firefox/41.0"
+)
+AMAZON_SEARCH_URL = "https://www.amazon.com/s?k={amazon_id}"
+
+
+class AmazonAttribute(Enum):
+    SERIES = 0
+    PAGES = 1
+    LANGUAGE = 2
+    PUBLISHER = 3
+    PUB_DATE = 4
+    DIMENSIONS = 5
+    ISBN_10 = 6
+    ISBN_13 = 7
+
+
+def strip_and_clean(text):
+    return text.strip("\n").rstrip().lstrip()
+
+
+def get_rating_from_soup(soup) -> Optional[int]:
+    rating = None
+    try:
+        potential_rating = soup.find("div", class_="allmusic-rating")
+        if potential_rating:
+            rating = int(strip_and_clean(potential_rating.get_text()))
+    except ValueError:
+        pass
+    return rating
+
+
+def get_review_from_soup(soup) -> str:
+    review = ""
+    try:
+        potential_text = soup.find("div", class_="text")
+        if potential_text:
+            review = strip_and_clean(potential_text.get_text())
+    except ValueError:
+        pass
+    return review
+
+
+def scrape_data_from_amazon(url) -> dict:
+    data_dict = {}
+    headers = {"User-Agent": USER_AGENT}
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "html.parser")
+        import pdb
+
+        pdb.set_trace()
+        data_dict["rating"] = get_rating_from_soup(soup)
+        data_dict["review"] = get_review_from_soup(soup)
+    return data_dict
+
+
+def get_amazon_product_dict(amazon_id: str) -> dict:
+    data_dict = {}
+    url = ""
+
+    search_url = AMAZON_SEARCH_URL.format(amazon_id=amazon_id)
+    headers = {
+        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+        "accept-language": "en-GB,en;q=0.9",
+    }
+
+    response = requests.get(search_url, headers=headers)
+
+    if response.status_code != 200:
+        logger.info(f"Bad http response from Amazon {response}")
+        return data_dict
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    results = soup.find("a", class_="a-link-normal")
+
+    if not results:
+        logger.info(f"No search results for {amazon_id}")
+        return data_dict
+
+    product_url = "https://www.amazon.com" + str(results.get("href", ""))
+
+    data_dict = {}
+    response = requests.get(product_url, headers=headers)
+
+    if response.status_code != 200:
+        logger.info(f"Bad http response from Amazon {response}")
+        return data_dict
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    try:
+        data_dict["title"] = soup.findAll("span", class_="celwidget")[
+            1
+        ].text.strip()
+        data_dict["cover_url"] = soup.find("img", class_="frontImage").get(
+            "src"
+        )
+        data_dict["summary"] = soup.findAll(
+            "div", class_="a-expander-content"
+        )[1].text
+        meta = soup.findAll("div", class_="rpi-attribute-value")
+        data_dict["isbn"] = meta[AmazonAttribute.ISBN_10.value].text.strip()
+        pages = meta[AmazonAttribute.PAGES.value].text
+        if "pages" in pages:
+            data_dict["pages"] = (
+                meta[AmazonAttribute.PAGES.value]
+                .text.split("pages")[0]
+                .strip()
+            )
+    except IndexError as e:
+        logger.error(
+            f"Amazon lookup is failing for this product {amazon_id}: {e}"
+        )
+    except AttributeError as e:
+        logger.error(
+            f"Amazon lookup is failing for this product {amazon_id}: {e}"
+        )
+
+    return data_dict
+
+
+def lookup_book_from_amazon(amazon_id: str) -> dict:
+    top = {}
+
+    return {
+        "title": top.get("title"),
+        "isbn": isbn,
+        "openlibrary_id": ol_id,
+        "goodreads_id": get_first("id_goodreads", top),
+        "first_publish_year": top.get("first_publish_year"),
+        "first_sentence": first_sentence,
+        "pages": top.get("number_of_pages_median", None),
+        "cover_url": COVER_URL.format(id=ol_id),
+        "ol_author_id": ol_author_id,
+        "subject_key_list": top.get("subject_key", []),
+    }

vrobbler/apps/books/koreader.py

@@ -91,14 +91,14 @@ def get_book_map_from_sqlite(rows: Iterable) -> dict:
                     if author_str == "N/A":
                         continue
 
-                    author, created = Author.objects.get_or_create(
-                        name=author_str
-                    )
-                    if created:
-                        author.openlibrary_id = get_author_openlibrary_id(
-                            author_str
+                    author = Author.objects.filter(name=author_str).first()
+                    if not author:
+                        author = Author.objects.create(
+                            name=author_str,
+                            openlibrary_id=get_author_openlibrary_id(
+                                author_str
+                            ),
                         )
-                        author.save(update_fields=["openlibrary_id"])
                         author.fix_metadata()
                         logger.debug(f"Created author {author}")
                     book.authors.add(author)
@@ -129,6 +129,8 @@ def build_scrobbles_from_pages(
     book_read_time_map = {}
     for page_row in rows:
         koreader_id = page_row[KoReaderPageStatColumn.ID_BOOK.value]
+        if koreader_id not in book_id_map.keys():
+            continue
         page_number = page_row[KoReaderPageStatColumn.PAGE.value]
         ts = page_row[KoReaderPageStatColumn.START_TIME.value]
         book_id = book_id_map[koreader_id][0]
@@ -168,6 +170,7 @@ def build_scrobbles_from_pages(
                     book_id=page.book_id,
                     user_id=user_id,
                     source="KOReader",
+                    media_type=Scrobble.MediaType.BOOK,
                     timestamp=page.start_time,
                     played_to_completion=True,
                     playback_position_seconds=playback_position_seconds,

vrobbler/apps/books/locg.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+from enum import Enum
+from typing import Optional
+from bs4 import BeautifulSoup
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+HEADERS = {
+    "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
+    "accept-language": "en-GB,en;q=0.9",
+}
+LOCG_WRTIER_URL = ""
+LOCG_WRITER_DETAIL_URL = "https://leagueofcomicgeeks.com/people/{slug}"
+LOCG_SEARCH_URL = (
+    "https://leagueofcomicgeeks.com/search/ajax_issues?query={query}"
+)
+LOCG_DETAIL_URL = "https://leagueofcomicgeeks.com/comic/{locg_slug}"
+
+
+def strip_and_clean(text):
+    return text.strip("\n").strip()
+
+
+def get_rating_from_soup(soup) -> Optional[int]:
+    rating = None
+    try:
+        potential_rating = soup.find("div", class_="allmusic-rating")
+        if potential_rating:
+            rating = int(strip_and_clean(potential_rating.get_text()))
+    except ValueError:
+        pass
+    return rating
+
+
+def lookup_comic_writer_by_locg_slug(slug: str) -> dict:
+    data_dict = {}
+    writer_url = LOCG_WRITER_DETAIL_URL.format(slug=slug)
+
+    response = requests.get(writer_url, headers=HEADERS)
+
+    if response.status_code != 200:
+        logger.info(f"Bad http response from LOCG {response}")
+        return data_dict
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    data_dict["locg_slug"] = slug
+    data_dict["name"] = soup.find("h1").text.strip()
+    data_dict["photo_url"] = soup.find("div", class_="avatar").img.get("src")
+
+    return data_dict
+
+
+def lookup_comic_by_locg_slug(slug: str) -> dict:
+    data_dict = {}
+    product_url = LOCG_DETAIL_URL.format(locg_slug=slug)
+
+    response = requests.get(product_url, headers=HEADERS)
+
+    if response.status_code != 200:
+        logger.info(f"Bad http response from LOCG {response}")
+        return data_dict
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    try:
+        data_dict["title"] = soup.find("h1").text.strip()
+        data_dict["summary"] = soup.find("p").text.strip()
+        data_dict["cover_url"] = (
+            soup.find("div", class_="cover-art").find("img").get("src")
+        )
+        attrs = soup.findAll("div", class_="details-addtl-block")
+        try:
+            data_dict["pages"] = (
+                attrs[1]
+                .find("div", class_="value")
+                .text.split("pages")[0]
+                .strip()
+            )
+        except IndexError:
+            logger.warn(f"No ISBN field")
+        try:
+            data_dict["isbn"] = (
+                attrs[3].find("div", class_="value").text.strip()
+            )
+        except IndexError:
+            logger.warn(f"No ISBN field")
+
+        writer_slug = None
+        try:
+            writer_slug = (
+                soup.findAll("div", class_="name")[5]
+                .a.get("href")
+                .split("people/")[1]
+            )
+        except IndexError:
+            logger.warn(f"No wrtier found")
+        if writer_slug:
+            data_dict["locg_writer_slug"] = writer_slug
+
+    except AttributeError:
+        logger.warn(f"Trouble parsing HTML, elements missing")
+
+    return data_dict
+
+
+def lookup_comic_from_locg(title: str) -> dict:
+    search_url = LOCG_SEARCH_URL.format(query=title)
+    response = requests.get(search_url, headers=HEADERS)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad http response from LOCG {response}")
+        return {}
+
+    soup = BeautifulSoup(response.text, "html.parser")
+
+    try:
+        slug = soup.findAll("a")[1].get("href").split("comic/")[1]
+    except IndexError:
+        logger.warn(f"No comic found on LOCG for {title}")
+        return {}
+
+    return lookup_comic_by_locg_slug(slug)

vrobbler/apps/books/migrations/0014_alter_book_genre.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-04-18 02:33
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+        ("books", "0013_page_user"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                blank=True,
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0015_alter_book_first_sentence_alter_book_genre.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.1.7 on 2023-08-19 02:47
+
+from django.db import migrations, models
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0043_scrobbledpage"),
+        ("books", "0014_alter_book_genre"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="first_sentence",
+            field=models.TextField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="book",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0016_book_locg_slug_book_summary.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.7 on 2023-08-26 04:48
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0015_alter_book_first_sentence_alter_book_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="locg_slug",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="summary",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0017_alter_book_authors.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.7 on 2023-08-26 16:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0016_book_locg_slug_book_summary"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="authors",
+            field=models.ManyToManyField(
+                blank=True, null=True, to="books.author"
+            ),
+        ),
+    ]

vrobbler/apps/books/migrations/0018_author_locg_slug.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-08-31 03:57
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0017_alter_book_authors"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="author",
+            name="locg_slug",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0019_alter_book_authors.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-11-21 23:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0018_author_locg_slug"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="authors",
+            field=models.ManyToManyField(blank=True, to="books.author"),
+        ),
+    ]

vrobbler/apps/books/models.py

@@ -13,8 +13,21 @@ from django.core.files.base import ContentFile
 from django.db import models
 from django.urls import reverse
 from django_extensions.db.models import TimeStampedModel
-from scrobbles.mixins import LongPlayScrobblableMixin, ScrobblableMixin
+from imagekit.models import ImageSpecField
+from imagekit.processors import ResizeToFit
+from scrobbles.mixins import (
+    LongPlayScrobblableMixin,
+    ObjectWithGenres,
+    ScrobblableMixin,
+)
 from scrobbles.utils import get_scrobbles_for_media
+from taggit.managers import TaggableManager
+
+from vrobbler.apps.books.locg import (
+    lookup_comic_by_locg_slug,
+    lookup_comic_from_locg,
+    lookup_comic_writer_by_locg_slug,
+)
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
@@ -26,9 +39,22 @@ class Author(TimeStampedModel):
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     openlibrary_id = models.CharField(max_length=255, **BNULL)
     headshot = models.ImageField(upload_to="books/authors/", **BNULL)
+    headshot_small = ImageSpecField(
+        source="headshot",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    headshot_medium = ImageSpecField(
+        source="headshot",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
     bio = models.TextField(**BNULL)
     wikipedia_url = models.CharField(max_length=255, **BNULL)
     isni = models.CharField(max_length=255, **BNULL)
+    locg_slug = models.CharField(max_length=255, **BNULL)
     wikidata_id = models.CharField(max_length=255, **BNULL)
     goodreads_id = models.CharField(max_length=255, **BNULL)
     librarything_id = models.CharField(max_length=255, **BNULL)
@@ -63,7 +89,7 @@ class Book(LongPlayScrobblableMixin):
     )
 
     title = models.CharField(max_length=255)
-    authors = models.ManyToManyField(Author)
+    authors = models.ManyToManyField(Author, blank=True)
     goodreads_id = models.CharField(max_length=255, **BNULL)
     koreader_id = models.IntegerField(**BNULL)
     koreader_authors = models.CharField(max_length=255, **BNULL)
@@ -72,12 +98,28 @@ class Book(LongPlayScrobblableMixin):
     pages = models.IntegerField(**BNULL)
     language = models.CharField(max_length=4, **BNULL)
     first_publish_year = models.IntegerField(**BNULL)
-    first_sentence = models.CharField(max_length=255, **BNULL)
+    first_sentence = models.TextField(**BNULL)
     openlibrary_id = models.CharField(max_length=255, **BNULL)
+    locg_slug = models.CharField(max_length=255, **BNULL)
     cover = models.ImageField(upload_to="books/covers/", **BNULL)
+    cover_small = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    cover_medium = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
+    summary = models.TextField(**BNULL)
+
+    genre = TaggableManager(through=ObjectWithGenres)
 
     def __str__(self):
-        return f"{self.title} by {self.author}"
+        return f"{self.title}"
 
     @property
     def subtitle(self):
@@ -87,7 +129,7 @@ class Book(LongPlayScrobblableMixin):
     def primary_image_url(self) -> str:
         url = ""
         if self.cover:
-            url = self.cover.url
+            url = self.cover_medium.url
         return url
 
     def get_start_url(self):
@@ -96,32 +138,65 @@ class Book(LongPlayScrobblableMixin):
     def get_absolute_url(self):
         return reverse("books:book_detail", kwargs={"slug": self.uuid})
 
-    def fix_metadata(self, force_update=False):
-        if not self.openlibrary_id or force_update:
+    def fix_metadata(self, data: dict = {}, force_update=False):
+        if (not self.openlibrary_id or not self.locg_slug) or force_update:
             author_name = ""
             if self.author:
                 author_name = self.author.name
-            book_dict = lookup_book_from_openlibrary(self.title, author_name)
-            if not book_dict:
-                logger.warn(f"Book not found in OL {self.title}")
-                return
 
-            cover_url = book_dict.pop("cover_url", "")
-            ol_author_id = book_dict.pop("ol_author_id", "")
-            ol_author_name = book_dict.pop("ol_author_name", "")
-            if book_dict.get("pages") == None:
-                book_dict.pop("pages")
+            if not data:
+                if self.locg_slug:
+                    data = lookup_comic_by_locg_slug(str(self.locg_slug))
+                else:
+                    data = lookup_comic_from_locg(str(self.title))
 
-            ol_title = book_dict.get("title", "")
+                if not data:
+                    logger.warn(
+                        f"Book not found on LOCG, checking OL {self.title}"
+                    )
+                    if self.openlibrary_id and force_update:
+                        data = lookup_book_from_openlibrary(
+                            str(self.openlibrary_id)
+                        )
+                    else:
+                        data = lookup_book_from_openlibrary(
+                            str(self.title), author_name
+                        )
+                if not data:
+                    logger.warn(f"Book not found in OL {self.title}")
+                    return
 
-            if ol_title.lower() != self.title.lower():
+            # We can discard the author name from OL for now, we'll lookup details below
+            data.pop("ol_author_name", "")
+            if data.get("ol_author_id"):
+                self.fix_authors_metadata(data.pop("ol_author_id", ""))
+            if data.get("locg_writer_slug"):
+                self.get_author_from_locg(data.pop("locg_writer_slug", ""))
+
+            ol_title = data.get("title", "")
+
+            # Kick out a little warning if we're about to change KoReader's title
+            if ol_title.lower() != str(self.title).lower():
                 logger.warn(
                     f"OL and KoReader disagree on this book title {self.title} != {ol_title}"
                 )
 
-            Book.objects.filter(pk=self.id).update(**book_dict)
+            # If we don't know pages, don't overwrite existing with None
+            if data.get("pages") == None:
+                data.pop("pages")
+
+            # Pop this, so we can look it up later
+            cover_url = data.pop("cover_url", "")
+
+            subject_key_list = data.pop("subject_key_list", "")
+
+            # Fun trick for updating all fields at once
+            Book.objects.filter(pk=self.id).update(**data)
             self.refresh_from_db()
 
+            if subject_key_list:
+                self.genre.add(*subject_key_list)
+
             if cover_url:
                 r = requests.get(cover_url)
                 if r.status_code == 200:
@@ -129,12 +204,44 @@ class Book(LongPlayScrobblableMixin):
                     self.cover.save(fname, ContentFile(r.content), save=True)
 
             if self.pages:
-                self.run_time_seconds = self.pages * int(
+                self.run_time_seconds = int(self.pages) * int(
                     self.AVG_PAGE_READING_SECONDS
                 )
 
             self.save()
 
+    def fix_authors_metadata(self, openlibrary_author_id):
+        author = Author.objects.filter(
+            openlibrary_id=openlibrary_author_id
+        ).first()
+        if not author:
+            data = lookup_author_from_openlibrary(openlibrary_author_id)
+            author_image_url = data.pop("author_headshot_url", None)
+
+            author = Author.objects.create(**data)
+
+            if author_image_url:
+                r = requests.get(author_image_url)
+                if r.status_code == 200:
+                    fname = f"{author.name}_{author.uuid}.jpg"
+                    author.headshot.save(
+                        fname, ContentFile(r.content), save=True
+                    )
+        self.authors.add(author)
+
+    def get_author_from_locg(self, locg_slug):
+        writer = lookup_comic_writer_by_locg_slug(locg_slug)
+
+        author, created = Author.objects.get_or_create(
+            name=writer["name"], locg_slug=writer["locg_slug"]
+        )
+        if (created or not author.headshot) and writer["photo_url"]:
+            r = requests.get(writer["photo_url"])
+            if r.status_code == 200:
+                fname = f"{author.name}_{author.uuid}.jpg"
+                author.headshot.save(fname, ContentFile(r.content), save=True)
+        self.authors.add(author)
+
     @property
     def author(self):
         return self.authors.first()
@@ -161,12 +268,24 @@ class Book(LongPlayScrobblableMixin):
         return progress
 
     @classmethod
-    def find_or_create(cls, data_dict: dict) -> "Game":
-        from books.utils import update_or_create_book
+    def find_or_create(cls, lookup_id: str, author: str = "") -> "Book":
+        data = lookup_book_from_openlibrary(lookup_id, author)
 
-        return update_or_create_book(
-            data_dict.get("title"), data_dict.get("author")
-        )
+        book, book_created = cls.objects.get_or_create(isbn=data["isbn"])
+        if book_created:
+            book.fix_metadata(data=data)
+
+        return book
+
+    def save(self, *args, **kwargs):
+        if (
+            (not self.isbn and not self.cover)
+            and (self.locg_slug or self.openlibrary_id)
+            and self.id
+        ):
+            self.fix_metadata(force_update=True)
+
+        return super(Book, self).save(*args, **kwargs)
 
 
 class Page(TimeStampedModel):

vrobbler/apps/books/openlibrary.py

@@ -1,5 +1,6 @@
 import json
 import logging
+from typing import Optional
 import urllib
 
 import requests
@@ -74,7 +75,9 @@ def lookup_author_from_openlibrary(olid: str) -> dict:
     }
 
 
-def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
+def lookup_book_from_openlibrary(
+    title: str, author: Optional[str] = None
+) -> dict:
     title_quoted = urllib.parse.quote(title)
     author_quoted = ""
     if author:
@@ -126,4 +129,5 @@ def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
         "pages": top.get("number_of_pages_median", None),
         "cover_url": COVER_URL.format(id=ol_id),
         "ol_author_id": ol_author_id,
+        "subject_key_list": top.get("subject_key", []),
     }

vrobbler/apps/books/utils.py

@@ -1,52 +0,0 @@
-from django.core.files.base import ContentFile
-import requests
-from typing import Optional
-from books.openlibrary import (
-    lookup_author_from_openlibrary,
-    lookup_book_from_openlibrary,
-)
-from books.models import Author, Book
-
-
-def update_or_create_book(
-    title: str, author: Optional[str] = None, force_update=False
-) -> Book:
-    book_dict = lookup_book_from_openlibrary(title, author)
-
-    book, book_created = Book.objects.get_or_create(
-        isbn=book_dict.get("isbn"),
-    )
-
-    if book_created or force_update:
-        cover_url = book_dict.pop("cover_url")
-        ol_author_id = book_dict.pop("ol_author_id")
-
-        Book.objects.filter(pk=book.id).update(**book_dict)
-        book.refresh_from_db()
-
-        # Process authors
-        author_dict = lookup_author_from_openlibrary(ol_author_id)
-        author = Author.objects.filter(openlibrary_id=ol_author_id).first()
-        if not author:
-            author_image_url = author_dict.pop("author_headshot_url", None)
-
-            author = Author.objects.create(**author_dict)
-
-            if author_image_url:
-                r = requests.get(author_image_url)
-                if r.status_code == 200:
-                    fname = f"{author.name}_{author.uuid}.jpg"
-                    author.headshot.save(
-                        fname, ContentFile(r.content), save=True
-                    )
-        book.authors.add(author)
-
-        # Process cover URL
-        r = requests.get(cover_url)
-        if r.status_code == 200:
-            fname = f"{book.title}_{book.uuid}.jpg"
-            book.cover.save(fname, ContentFile(r.content), save=True)
-
-        book.fix_metadata()
-
-    return book

vrobbler/apps/locations/admin.py

@@ -0,0 +1,36 @@
+from django.contrib import admin
+
+from locations.models import GeoLocation, RawGeoLocation
+
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(GeoLocation)
+class GeoLocationAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "created",
+        "lat",
+        "lon",
+        "title",
+        "altitude",
+    )
+    ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]
+
+
+@admin.register(RawGeoLocation)
+class RawGeoLocationAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "lat",
+        "lon",
+        "altitude",
+        "speed",
+    )
+    ordering = (
+        "lat",
+        "lon",
+    )

vrobbler/apps/locations/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class LocationConfig(AppConfig):
+    name = "locations"
+

vrobbler/apps/locations/constants.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+from collections import defaultdict
+
+
+LOCATION_PROVIDERS = defaultdict(lambda: "Unknown")
+LOCATION_PROVIDERS["gps"] = "GPS"
+LOCATION_PROVIDERS["network"] = "Wifi Triangulation"

vrobbler/apps/locations/migrations/0001_initial.py

@@ -0,0 +1,77 @@
+# Generated by Django 4.1.7 on 2023-11-21 23:29
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import taggit.managers
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    operations = [
+        migrations.CreateModel(
+            name="GeoLocation",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "title",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "run_time_seconds",
+                    models.IntegerField(blank=True, null=True),
+                ),
+                (
+                    "run_time_ticks",
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ("lat", models.FloatField()),
+                ("lon", models.FloatField()),
+                ("altitude", models.FloatField(blank=True, null=True)),
+                (
+                    "genre",
+                    taggit.managers.TaggableManager(
+                        blank=True,
+                        help_text="A comma-separated list of tags.",
+                        through="scrobbles.ObjectWithGenres",
+                        to="scrobbles.Genre",
+                        verbose_name="Tags",
+                    ),
+                ),
+            ],
+            options={
+                "unique_together": {("lat", "lon", "altitude")},
+            },
+        ),
+    ]

vrobbler/apps/locations/migrations/0002_rawgeolocation.py

@@ -0,0 +1,58 @@
+# Generated by Django 4.1.7 on 2023-11-22 00:13
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("locations", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RawGeoLocation",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("lat", models.FloatField()),
+                ("lon", models.FloatField()),
+                ("altitude", models.FloatField(blank=True, null=True)),
+                ("speed", models.FloatField(blank=True, null=True)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

vrobbler/apps/locations/migrations/0003_rawgeolocation_timestamp.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-11-22 23:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("locations", "0002_rawgeolocation"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="rawgeolocation",
+            name="timestamp",
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/locations/migrations/0004_alter_rawgeolocation_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.1.7 on 2023-11-24 12:45
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("locations", "0003_rawgeolocation_timestamp"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="rawgeolocation",
+            options={},
+        ),
+    ]

vrobbler/apps/locations/migrations/0005_alter_rawgeolocation_options_and_more.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.1.7 on 2023-11-24 18:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("locations", "0004_alter_rawgeolocation_options"),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name="rawgeolocation",
+            options={"get_latest_by": "modified"},
+        ),
+        migrations.AddField(
+            model_name="geolocation",
+            name="truncated_lat",
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="geolocation",
+            name="truncated_lon",
+            field=models.FloatField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/locations/models.py

@@ -0,0 +1,96 @@
+import logging
+from typing import Dict
+from uuid import uuid4
+
+from django.contrib.auth import get_user_model
+from django.conf import settings
+from django.db import models
+from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
+from scrobbles.mixins import ScrobblableMixin
+
+logger = logging.getLogger(__name__)
+BNULL = {"blank": True, "null": True}
+User = get_user_model()
+
+
+class GeoLocation(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, "LOCATION_COMPLETION_PERCENT", 100)
+
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    lat = models.FloatField()
+    lon = models.FloatField()
+    truncated_lat = models.FloatField(**BNULL)
+    truncated_lon = models.FloatField(**BNULL)
+    altitude = models.FloatField(**BNULL)
+
+    class Meta:
+        unique_together = [["lat", "lon", "altitude"]]
+
+    def __str__(self):
+        if self.title:
+            return self.title
+
+        return f"{self.lat} x {self.lon}"
+
+    def get_absolute_url(self):
+        return reverse(
+            "locations:geo_location_detail", kwargs={"slug": self.uuid}
+        )
+
+    @classmethod
+    def find_or_create(cls, data_dict: Dict) -> "GeoLocation":
+        """Given a data dict from GPSLogger, does the heavy lifting of looking up
+        the location, creating if if doesn't exist yet.
+
+        """
+        # TODO Add constants for all these data keys
+        if "lat" not in data_dict.keys() or "lon" not in data_dict.keys():
+            logger.error("No lat or lon keys in data dict")
+            return
+
+        int_lat, r_lat = str(data_dict.get("lat", "")).split(".")
+        int_lon, r_lon = str(data_dict.get("lon", "")).split(".")
+
+        try:
+            trunc_lat = r_lat[0:4]
+        except IndexError:
+            trunc_lat = r_lat
+        try:
+            trunc_lon = r_lon[0:4]
+        except IndexError:
+            trunc_lon = r_lon
+
+        data_dict["lat"] = float(f"{int_lat}.{trunc_lat}")
+        data_dict["lon"] = float(f"{int_lon}.{trunc_lon}")
+
+        int_alt, r_alt = str(data_dict.get("alt", "")).split(".")
+        try:
+            trunc_alt = r_lon[0:4]
+        except IndexError:
+            trunc_alt = r_alt
+
+        data_dict["altitude"] = float(f"{int_alt}.{trunc_alt}")
+
+        location = cls.objects.filter(
+            lat=data_dict.get("lat"),
+            lon=data_dict.get("lon"),
+            altitude=data_dict.get("altitude"),
+        ).first()
+
+        if not location:
+            location = cls.objects.create(
+                lat=data_dict.get("lat"),
+                lon=data_dict.get("lon"),
+                altitude=data_dict.get("altitude"),
+            )
+        return location
+
+
+class RawGeoLocation(TimeStampedModel):
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    lat = models.FloatField()
+    lon = models.FloatField()
+    altitude = models.FloatField(**BNULL)
+    speed = models.FloatField(**BNULL)
+    timestamp = models.DateTimeField(**BNULL)

vrobbler/apps/locations/urls.py

@@ -0,0 +1,18 @@
+from django.urls import path
+from locations import views
+
+app_name = "locations"
+
+
+urlpatterns = [
+    path(
+        "locations/",
+        views.GeoLocationListView.as_view(),
+        name="geo_locations_list",
+    ),
+    path(
+        "locations/<slug:slug>/",
+        views.GeoLocationDetailView.as_view(),
+        name="geo_location_detail",
+    ),
+]

vrobbler/apps/locations/views.py

@@ -0,0 +1,22 @@
+from django.db.models import Count
+from django.views import generic
+from locations.models import GeoLocation
+from scrobbles.stats import get_scrobble_count_qs
+
+
+class GeoLocationListView(generic.ListView):
+    model = GeoLocation
+    paginate_by = 75
+
+    def get_queryset(self):
+        return super().get_queryset().filter(scrobble__user_id=self.request.user.id).order_by("-scrobble__timestamp")
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        context_data["latest"] = self.get_queryset().first()
+        return context_data
+
+
+class GeoLocationDetailView(generic.DetailView):
+    model = GeoLocation
+    slug_field = "uuid"

vrobbler/apps/music/aggregators.py

@@ -26,7 +26,9 @@ def scrobble_counts(user=None):
     starting_day_of_current_year = now.date().replace(month=1, day=1)
 
     finished_scrobbles_qs = Scrobble.objects.filter(
-        user_filter, played_to_completion=True
+        user_filter,
+        played_to_completion=True,
+        media_type=Scrobble.MediaType.TRACK,
     )
     data = {}
     data["today"] = finished_scrobbles_qs.filter(

vrobbler/apps/music/lastfm.py

@@ -68,6 +68,7 @@ class LastFM:
                 track=track,
                 played_to_completion=True,
                 in_progress=False,
+                media_type=Scrobble.MediaType.TRACK,
             )
             # Vrobbler scrobbles on finish, LastFM scrobbles on start
             seconds_eariler = timestamp - timedelta(seconds=20)
@@ -107,6 +108,7 @@ class LastFM:
         # TOOD spin this out into a celery task over certain threshold of found scrobbles?
 
         for scrobble in found_scrobbles:
+            logger.debug(f"Processing {scrobble}")
             run_time = None
             mbid = None
             artist = None
@@ -121,6 +123,10 @@ class LastFM:
                 logger.warn(
                     "LastFM barfed trying to get the track for {scrobble.track}"
                 )
+            except pylast.NetworkError as e:
+                logger.warn(
+                    "LastFM barfed trying to get the track for {scrobble.track}"
+                )
 
             if not artist:
                 logger.warn(f"Silly LastFM, no artist found for {scrobble}")

vrobbler/apps/music/migrations/0023_alter_track_genre.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-04-18 02:33
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+        ("music", "0022_artist_theaudiodb_id"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="track",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                blank=True,
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/music/models.py

@@ -12,6 +12,8 @@ from django.db import models
 from django.urls import reverse
 from django.utils.translation import gettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
+from imagekit.models import ImageSpecField
+from imagekit.processors import ResizeToFit
 from music.allmusic import get_allmusic_slug, scrape_data_from_allmusic
 from music.bandcamp import get_bandcamp_slug
 from music.theaudiodb import lookup_album_from_tadb, lookup_artist_from_tadb
@@ -32,6 +34,18 @@ class Artist(TimeStampedModel):
     allmusic_id = models.CharField(max_length=100, **BNULL)
     bandcamp_id = models.CharField(max_length=100, **BNULL)
     thumbnail = models.ImageField(upload_to="artist/", **BNULL)
+    thumbnail_small = ImageSpecField(
+        source="thumbnail",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    thumbnail_medium = ImageSpecField(
+        source="thumbnail",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
 
     class Meta:
         unique_together = [["name", "musicbrainz_id"]]
@@ -39,6 +53,14 @@ class Artist(TimeStampedModel):
     def __str__(self):
         return self.name
 
+    @property
+    def primary_image_url(self) -> str:
+        if self.thumbnail:
+            return self.thumbnail.url
+        if self.album_set.first().cover_image:
+            return self.album_set.first().cover_image.url
+        return ""
+
     @property
     def mb_link(self):
         return f"https://musicbrainz.org/artist/{self.musicbrainz_id}"
@@ -96,10 +118,18 @@ class Artist(TimeStampedModel):
             self.bandcamp_id = slug
             self.save(update_fields=["bandcamp_id"])
 
-    def fix_metadata(self):
-        tadb_info = lookup_artist_from_tadb(self.name)
+    def fix_metadata(self, force_update=False):
+        tadb_info = {}
+        if self.theaudiodb_id and force_update:
+            tadb_info = lookup_artist_from_tadb(self.theaudiodb_id)
+
+        if not self.theaudiodb_id or (force_update and not tadb_info):
+            tadb_info = lookup_artist_from_tadb(self.name)
+
         if not tadb_info:
-            logger.warn(f"No response from TADB for artist {self.name}")
+            logger.warn(
+                f"No response from TADB for artist {self.name}, try force_update=True"
+            )
             return
 
         self.biography = tadb_info["biography"]
@@ -136,6 +166,18 @@ class Album(TimeStampedModel):
     musicbrainz_releasegroup_id = models.CharField(max_length=255, **BNULL)
     musicbrainz_albumartist_id = models.CharField(max_length=255, **BNULL)
     cover_image = models.ImageField(upload_to="albums/", **BNULL)
+    cover_image_small = ImageSpecField(
+        source="cover_image",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    cover_image_medium = ImageSpecField(
+        source="cover_image",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
     theaudiodb_id = models.CharField(max_length=255, unique=True, **BNULL)
     theaudiodb_description = models.TextField(**BNULL)
     theaudiodb_year_released = models.IntegerField(**BNULL)
@@ -168,6 +210,12 @@ class Album(TimeStampedModel):
             track__in=self.track_set.all()
         ).order_by("-timestamp")
 
+    @property
+    def primary_image_url(self) -> str:
+        if self.cover_image:
+            return self.cover_image_medium.url
+        return ""
+
     @property
     def tracks(self):
         return (
@@ -392,9 +440,9 @@ class Track(ScrobblableMixin):
     def primary_image_url(self) -> str:
         url = ""
         if self.artist.thumbnail:
-            url = self.artist.thumbnail.url
+            url = self.artist.thumbnail_medium.url
         if self.album and self.album.cover_image:
-            url = self.album.cover_image.url
+            url = self.album.cover_image_medium.url
         return url
 
     @classmethod

vrobbler/apps/music/musicbrainz.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Iterable
 
 import musicbrainzngs
 from dateutil.parser import parse
@@ -52,10 +53,18 @@ def lookup_album_from_mb(musicbrainz_id: str) -> dict:
 def lookup_album_dict_from_mb(release_name: str, artist_name: str) -> dict:
     musicbrainzngs.set_useragent("vrobbler", "0.3.0")
 
-    top_result = musicbrainzngs.search_releases(
-        release_name, artist=artist_name
-    )["release-list"][0]
-    score = int(top_result.get("ext:score"))
+    top_result = {}
+
+    try:
+        top_result = musicbrainzngs.search_releases(
+            release_name, artist=artist_name
+        )["release-list"][0]
+    except IndexError:
+        logger.info(
+            f"No release found on MB for {artist_name} and {release_name}"
+        )
+
+    score = int(top_result.get("ext:score", 0))
     if score < 85:
         logger.debug(
             "Album lookup score below 85 threshold",
@@ -74,19 +83,22 @@ def lookup_album_dict_from_mb(release_name: str, artist_name: str) -> dict:
     }
 
 
-def lookup_artist_from_mb(artist_name: str) -> str:
+def lookup_artist_from_mb(artist_name: str) -> dict:
     musicbrainzngs.set_useragent("vrobbler", "0.3.0")
 
-    top_result = musicbrainzngs.search_artists(artist=artist_name)[
-        "artist-list"
-    ][0]
+    try:
+        top_result = musicbrainzngs.search_artists(artist=artist_name)[
+            "artist-list"
+        ][0]
+    except IndexError:
+        return {}
     score = int(top_result.get("ext:score"))
     if score < 85:
         logger.debug(
             "Artist lookup score below 85 threshold",
             extra={"result": top_result},
         )
-        return ""
+        return {}
 
     return top_result
 

vrobbler/apps/music/theaudiodb.py

@@ -7,23 +7,48 @@ from django.conf import settings
 
 THEAUDIODB_API_KEY = getattr(settings, "THEAUDIODB_API_KEY")
 ARTIST_SEARCH_URL = f"https://www.theaudiodb.com/api/v1/json/{THEAUDIODB_API_KEY}/search.php?s="
+ARTIST_FETCH_URL = f"https://www.theaudiodb.com/api/v1/json/{THEAUDIODB_API_KEY}/artist.php?i="
 ALBUM_SEARCH_URL = f"https://www.theaudiodb.com/api/v1/json/{THEAUDIODB_API_KEY}/searchalbum.php?s="
 
 logger = logging.getLogger(__name__)
 
 
-def lookup_artist_from_tadb(name: str) -> dict:
+def lookup_artist_from_tadb(name_or_id: str) -> dict:
     artist_info = {}
-    name = urllib.parse.quote(name)
-    response = requests.get(ARTIST_SEARCH_URL + name)
+    response = None
+
+    # Look for an ID as an integer
+    try:
+        tadb_id = int(name_or_id)
+    except ValueError:
+        tadb_id = None
+
+    if tadb_id:
+        response = requests.get(ARTIST_FETCH_URL + str(tadb_id))
+
+        if response.status_code != 200:
+            logger.warn(f"Bad response from TADB: {response.status_code}")
+            return artist_info
+
+        if not response.content:
+            logger.warn(f"Bad content from TADB: {response.content}")
+            return artist_info
+
+    if not response:
+        name = urllib.parse.quote(name_or_id)
+        response = requests.get(ARTIST_SEARCH_URL + name)
 
     if response.status_code != 200:
         logger.warn(f"Bad response from TADB: {response.status_code}")
-        return {}
+        return artist_info
 
     if not response.content:
         logger.warn(f"Bad content from TADB: {response.content}")
-        return {}
+        return artist_info
+
+    if '{"artists": null}' in str(response.content):
+        logger.warn(f"Bad content from TADB: {response.content}")
+        return artist_info
 
     results = json.loads(response.content)
     if results["artists"]:

vrobbler/apps/music/utils.py

@@ -26,9 +26,10 @@ def get_or_create_artist(name: str, mbid: str = None) -> Artist:
         name = re.split("&", name, flags=re.IGNORECASE)[0].strip()
 
     artist_dict = lookup_artist_from_mb(name)
-    mbid = mbid or artist_dict["id"]
+    mbid = mbid or artist_dict.get("id", None)
 
-    artist = Artist.objects.filter(musicbrainz_id=mbid).first()
+    if mbid:
+        artist = Artist.objects.filter(musicbrainz_id=mbid).first()
     if not artist:
         artist = Artist.objects.create(name=name, musicbrainz_id=mbid)
         artist.fix_metadata()
@@ -42,20 +43,34 @@ def get_or_create_album(
     album = None
     album_dict = lookup_album_dict_from_mb(name, artist_name=artist.name)
 
-    name = name or album_dict["title"]
-    album = Album.objects.filter(artists__in=[artist], name=name).first()
+    name = name or album_dict.get("title", None)
+    if not name:
+        logger.debug(
+            f"Cannot get or create album by {artist} with no name ({name})"
+        )
+        return
+
+    album = Album.objects.filter(
+        musicbrainz_id=mbid, name=name, artists__in=[artist]
+    ).first()
+
+    if not album:
+        mbid_group = album_dict.get("mb_group_id")
+        album = Album.objects.filter(
+            musicbrainz_releasegroup_id=mbid_group
+        ).first()
 
     if not album and name:
         mbid = mbid or album_dict["mb_id"]
-        album, album_created = Album.objects.get_or_create(
-            name=name, musicbrainz_id=mbid
-        )
+        album, album_created = Album.objects.get_or_create(musicbrainz_id=mbid)
         if album_created:
+            album.name = name
             album.year = album_dict["year"]
             album.musicbrainz_releasegroup_id = album_dict["mb_group_id"]
             album.musicbrainz_albumartist_id = artist.musicbrainz_id
             album.save(
                 update_fields=[
+                    "name",
                     "musicbrainz_id",
                     "year",
                     "musicbrainz_releasegroup_id",
@@ -77,12 +92,12 @@ def get_or_create_album(
 def get_or_create_track(
     title: str,
     artist: Artist,
-    album: Album,
+    album: Album = None,
     mbid: str = None,
     run_time_seconds=None,
 ) -> Track:
     track = None
-    if not mbid:
+    if not mbid and album:
         mbid = lookup_track_from_mb(
             title,
             artist.musicbrainz_id,

vrobbler/apps/podcasts/migrations/0012_alter_episode_genre.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-04-18 02:33
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+        ("podcasts", "0011_rename_url_podcast_feed_url_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="episode",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                blank=True,
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/podcasts/scrapers.py

@@ -1,4 +1,3 @@
-import urllib
 from typing import Optional
 from bs4 import BeautifulSoup
 import requests

vrobbler/apps/profiles/migrations/0008_userprofile_lastfm_auto_import.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-04-06 17:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0007_alter_userprofile_timezone"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="lastfm_auto_import",
+            field=models.BooleanField(default=False),
+        ),
+    ]

vrobbler/apps/profiles/migrations/0009_userprofile_retroarch_auto_import_and_more.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.7 on 2023-05-25 02:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0008_userprofile_lastfm_auto_import"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="retroarch_auto_import",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="userprofile",
+            name="retroarch_path",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/profiles/migrations/0010_userprofile_redirect_to_webpage.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-12-11 19:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("profiles", "0009_userprofile_retroarch_auto_import_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="userprofile",
+            name="redirect_to_webpage",
+            field=models.BooleanField(default=True),
+        ),
+    ]

vrobbler/apps/profiles/models.py

@@ -23,6 +23,12 @@ class UserProfile(TimeStampedModel):
     )
     lastfm_username = models.CharField(max_length=255, **BNULL)
     lastfm_password = EncryptedField(**BNULL)
+    lastfm_auto_import = models.BooleanField(default=False)
+
+    retroarch_path = models.CharField(max_length=255, **BNULL)
+    retroarch_auto_import = models.BooleanField(default=False)
+
+    redirect_to_webpage = models.BooleanField(default=True)
 
     def __str__(self):
         return f"User profile for {self.user}"

vrobbler/apps/scrobbles/admin.py

@@ -1,10 +1,13 @@
 from django.contrib import admin
+
 from scrobbles.models import (
     AudioScrobblerTSVImport,
     ChartRecord,
     KoReaderImport,
     LastFmImport,
+    RetroarchImport,
     Scrobble,
+    ScrobbledPage,
 )
 from scrobbles.mixins import Genre
 
@@ -19,6 +22,9 @@ class ScrobbleInline(admin.TabularInline):
         "video_game",
         "book",
         "sport_event",
+        "board_game",
+        "geo_location",
+        "webpage",
         "user",
     )
     exclude = ("source_id", "scrobble_log")
@@ -37,17 +43,22 @@ class ImportBaseAdmin(admin.ModelAdmin):
 
 @admin.register(AudioScrobblerTSVImport)
 class AudioScrobblerTSVImportAdmin(ImportBaseAdmin):
-    """"""
+    ...
 
 
 @admin.register(LastFmImport)
 class LastFmImportAdmin(ImportBaseAdmin):
-    """"""
+    ...
 
 
 @admin.register(KoReaderImport)
 class KoReaderImportAdmin(ImportBaseAdmin):
-    """"""
+    ...
+
+
+@admin.register(RetroarchImport)
+class RetroarchImportAdmin(ImportBaseAdmin):
+    ...
 
 
 @admin.register(Genre)
@@ -101,6 +112,7 @@ class ScrobbleAdmin(admin.ModelAdmin):
     list_filter = (
         "is_paused",
         "in_progress",
+        "media_type",
         "long_play_complete",
         "source",
     )
@@ -109,8 +121,15 @@ class ScrobbleAdmin(admin.ModelAdmin):
     def media_name(self, obj):
         return obj.media_obj
 
-    def media_type(self, obj):
-        return obj.media_obj.__class__.__name__
-
     def playback_percent(self, obj):
         return obj.percent_played
+
+
+@admin.register(ScrobbledPage)
+class ScrobbledPageAdmin(admin.ModelAdmin):
+    list_display = (
+        "number",
+        "scrobble",
+        "notes",
+    )
+    raw_id_fields = ("scrobble",)

vrobbler/apps/scrobbles/constants.py

@@ -8,6 +8,24 @@ LONG_PLAY_MEDIA = {
     "books": "Book",
 }
 
+PLAY_AGAIN_MEDIA = {
+    "videogames": "VideoGame",
+    "books": "Book",
+    "boardgames": "BoardGame",
+}
+
+
+EXCLUDE_FROM_NOW_PLAYING = ("GeoLocation",)
+
+MANUAL_SCROBBLE_FNS = {
+    "-v": "manual_scrobble_video_game",
+    "-b": "manual_scrobble_book",
+    "-s": "manual_scrobble_event",
+    "-i": "manual_scrobble_video",
+    "-g": "manual_scrobble_board_game",
+    "-w": "manual_scrobble_webpage",
+}
+
 
 class AsTsvColumn(Enum):
     ARTIST_NAME = 0

vrobbler/apps/scrobbles/context_processors.py

@@ -1,7 +1,8 @@
 import pytz
 from django.utils import timezone
-from scrobbles.models import Scrobble
 
+from scrobbles.constants import EXCLUDE_FROM_NOW_PLAYING
+from scrobbles.models import Scrobble
 
 def now_playing(request):
     user = request.user
@@ -13,5 +14,7 @@ def now_playing(request):
             in_progress=True,
             is_paused=False,
             user=user,
+        ).exclude(
+            media_type__in=EXCLUDE_FROM_NOW_PLAYING,
         )
     }

vrobbler/apps/scrobbles/exceptions.py

@@ -0,0 +1,2 @@
+class UserNotFound(Exception):
+    ...

vrobbler/apps/scrobbles/management/commands/delete_zombie_scrobbles.py

@@ -0,0 +1,29 @@
+from django.core.management.base import BaseCommand, no_translations
+from vrobbler.apps.scrobbles.utils import delete_zombie_scrobbles
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--delete",
+            action="store_true",
+            help="Delete poll instead of closing it",
+        )
+
+    def handle(self, *args, **options):
+        dry_run = True
+        if options["delete"]:
+            dry_run = False
+        scrobbles_found = delete_zombie_scrobbles(dry_run)
+
+        if not scrobbles_found:
+            print(f"No zombie scrobbles found to delete")
+            return
+
+        if not dry_run:
+            print(f"Deleted {scrobbles_found} zombie scrobbles")
+            return
+
+        print(
+            f"Found {scrobbles_found} zombie scrobbles, use --delete to remove them"
+        )

vrobbler/apps/scrobbles/management/commands/import_from_lastfm.py

@@ -0,0 +1,18 @@
+from django.core.management.base import BaseCommand
+from vrobbler.apps.scrobbles.utils import import_lastfm_for_all_users
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--restart",
+            action="store_true",
+            help="Restart failed imports",
+        )
+
+    def handle(self, *args, **options):
+        restart = False
+        if options["restart"]:
+            restart = True
+        count = import_lastfm_for_all_users(restart=restart)
+        print(f"Started {count} LastFM imports")

vrobbler/apps/scrobbles/management/commands/import_from_retroarch.py

@@ -0,0 +1,18 @@
+from django.core.management.base import BaseCommand
+from vrobbler.apps.scrobbles.utils import import_retroarch_for_all_users
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--restart",
+            action="store_true",
+            help="Restart failed imports",
+        )
+
+    def handle(self, *args, **options):
+        restart = False
+        if options["restart"]:
+            restart = True
+        count = import_retroarch_for_all_users(restart=restart)
+        print(f"Started {count} Retroarch imports")

vrobbler/apps/scrobbles/management/commands/migrate_page_to_scrobbledpage.py

@@ -0,0 +1,63 @@
+from books.models import Book
+from django.core.management.base import BaseCommand, no_translations
+from scrobbles.models import ScrobbledPage
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--commit",
+            action="store_true",
+            help="Actually populate data",
+        )
+
+    def handle(self, *args, **options):
+        dry_run = True
+        if options["commit"]:
+            dry_run = False
+
+        pages_to_create = []
+        associated_scrobble = None
+        last_scrobble = None
+        for book in Book.objects.all():
+            for page in book.page_set.all().order_by("number"):
+                notes = ""
+                last_scrobble = associated_scrobble
+                if (
+                    not associated_scrobble
+                    or page.number > associated_scrobble.book_pages_read
+                ):
+                    associated_scrobble = page.user.scrobble_set.filter(
+                        book=page.book,
+                        timestamp__gte=page.start_time,
+                        timestamp__lte=page.end_time,
+                    ).first()
+
+                if (
+                    last_scrobble
+                    and not associated_scrobble
+                    and page.number > last_scrobble.book_pages_read
+                ):
+                    associated_scrobble = last_scrobble
+                    notes = f"Extrapolated reading from scrobble {associated_scrobble.id}"
+
+                pages_to_create.append(
+                    ScrobbledPage(
+                        scrobble=associated_scrobble,
+                        number=page.number,
+                        start_time=page.start_time,
+                        end_time=page.end_time,
+                        duration_seconds=page.duration_seconds,
+                        notes=notes,
+                    )
+                )
+
+        pages_to_move_len = len(pages_to_create)
+        if dry_run:
+            print(
+                f"Found {pages_to_move_len} to migrate. Use --commit to move them"
+            )
+            return
+
+        ScrobbledPage.objects.bulk_create(pages_to_create)
+        print(f"Migrated {pages_to_move_len} generic pages to scrobbled pages")

vrobbler/apps/scrobbles/migrations/0037_scrobble_media_type.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.1.7 on 2023-04-09 04:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0036_scrobble_stop_timestamp"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scrobble",
+            name="media_type",
+            field=models.CharField(
+                choices=[
+                    ("Video", "Video"),
+                    ("Track", "Track"),
+                    ("Episode", "Podcast episode"),
+                    ("SportEvent", "Sport event"),
+                    ("Book", "Book"),
+                    ("VideoGame", "Video game"),
+                ],
+                default="Video",
+                max_length=14,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0038_alter_objectwithgenres_tag.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.7 on 2023-04-17 22:01
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0037_scrobble_media_type"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="objectwithgenres",
+            name="tag",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="%(app_label)s_%(class)s_items",
+                to="scrobbles.genre",
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0039_scrobble_board_game.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.7 on 2023-04-17 22:11
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0001_initial"),
+        ("scrobbles", "0038_alter_objectwithgenres_tag"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scrobble",
+            name="board_game",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="boardgames.boardgame",
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0040_alter_scrobble_media_type.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.1.7 on 2023-04-18 02:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0039_scrobble_board_game"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="scrobble",
+            name="media_type",
+            field=models.CharField(
+                choices=[
+                    ("Video", "Video"),
+                    ("Track", "Track"),
+                    ("Episode", "Podcast episode"),
+                    ("SportEvent", "Sport event"),
+                    ("Book", "Book"),
+                    ("VideoGame", "Video game"),
+                    ("BoardGame", "Board game"),
+                ],
+                default="Video",
+                max_length=14,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0041_retroarchimport.py

@@ -0,0 +1,67 @@
+# Generated by Django 4.1.7 on 2023-05-25 02:17
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="RetroarchImport",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("uuid", models.UUIDField(default=uuid.uuid4, editable=False)),
+                (
+                    "processing_started",
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                (
+                    "processed_finished",
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                ("process_log", models.TextField(blank=True, null=True)),
+                ("process_count", models.IntegerField(blank=True, null=True)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Retroarch Import",
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0042_scrobble_videogame_screenshot.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.1.7 on 2023-06-03 18:18
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0041_retroarchimport"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scrobble",
+            name="videogame_screenshot",
+            field=models.ImageField(
+                blank=True,
+                null=True,
+                upload_to="scrobbles/videogame_screenshot/",
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0043_scrobbledpage.py

@@ -0,0 +1,63 @@
+# Generated by Django 4.1.7 on 2023-07-16 05:23
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0042_scrobble_videogame_screenshot"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ScrobbledPage",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("number", models.IntegerField()),
+                ("start_time", models.DateTimeField(blank=True, null=True)),
+                ("end_time", models.DateTimeField(blank=True, null=True)),
+                (
+                    "duration_seconds",
+                    models.IntegerField(blank=True, null=True),
+                ),
+                (
+                    "notes",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "scrobble",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to="scrobbles.scrobble",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0044_scrobble_geo_location_alter_scrobble_media_type.py

@@ -0,0 +1,43 @@
+# Generated by Django 4.1.7 on 2023-11-21 23:43
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("locations", "0001_initial"),
+        ("scrobbles", "0043_scrobbledpage"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scrobble",
+            name="geo_location",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="locations.geolocation",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="scrobble",
+            name="media_type",
+            field=models.CharField(
+                choices=[
+                    ("Video", "Video"),
+                    ("Track", "Track"),
+                    ("Episode", "Podcast episode"),
+                    ("SportEvent", "Sport event"),
+                    ("Book", "Book"),
+                    ("VideoGame", "Video game"),
+                    ("BoardGame", "Board game"),
+                    ("GeoLocation", "GeoLocation"),
+                ],
+                default="Video",
+                max_length=14,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0045_scrobble_webpage_alter_scrobble_media_type.py

@@ -0,0 +1,44 @@
+# Generated by Django 4.1.7 on 2023-11-30 16:45
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("webpages", "0001_initial"),
+        ("scrobbles", "0044_scrobble_geo_location_alter_scrobble_media_type"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="scrobble",
+            name="webpage",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="webpages.webpage",
+            ),
+        ),
+        migrations.AlterField(
+            model_name="scrobble",
+            name="media_type",
+            field=models.CharField(
+                choices=[
+                    ("Video", "Video"),
+                    ("Track", "Track"),
+                    ("Episode", "Podcast episode"),
+                    ("SportEvent", "Sport event"),
+                    ("Book", "Book"),
+                    ("VideoGame", "Video game"),
+                    ("BoardGame", "Board game"),
+                    ("GeoLocation", "GeoLocation"),
+                    ("WebPage", "Web Page"),
+                ],
+                default="Video",
+                max_length=14,
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/mixins.py

@@ -27,6 +27,7 @@ class ObjectWithGenres(GenericTaggedItemBase):
         Genre,
         on_delete=models.CASCADE,
         related_name="%(app_label)s_%(class)s_items",
+        **BNULL,
     )
 
 
@@ -38,7 +39,7 @@ class ScrobblableMixin(TimeStampedModel):
     run_time_seconds = models.IntegerField(**BNULL)
     run_time_ticks = models.PositiveBigIntegerField(**BNULL)
 
-    genre = TaggableManager(through=ObjectWithGenres)
+    genre = TaggableManager(through=ObjectWithGenres, blank=True)
 
     class Meta:
         abstract = True
@@ -55,6 +56,11 @@ class ScrobblableMixin(TimeStampedModel):
     def find_or_create(cls):
         logger.warn("find_or_create() not implemented yet")
 
+    def scrobble(self, user_id, **kwargs):
+        """Given a user ID and a dictionary of data, attempts to scrobble it"""
+        from scrobbles.models import Scrobble
+        Scrobble.create_or_update(self, user_id, **kwargs)
+
 
 class LongPlayScrobblableMixin(ScrobblableMixin):
     class Meta:

vrobbler/apps/scrobbles/models.py

@@ -4,13 +4,18 @@ import logging
 from typing import Optional
 from uuid import uuid4
 
+from boardgames.models import BoardGame
 from books.models import Book
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
+from django.utils.functional import cached_property
 from django_extensions.db.models import TimeStampedModel
+from imagekit.models import ImageSpecField
+from imagekit.processors import ResizeToFit
+from locations.models import GeoLocation
 from music.lastfm import LastFM
 from music.models import Artist, Track
 from podcasts.models import Episode
@@ -29,8 +34,10 @@ from scrobbles.utils import (
     check_scrobble_for_finish,
 )
 from sports.models import SportEvent
+from videogames import retroarch
 from videogames.models import VideoGame
 from videos.models import Series, Video
+from webpages.models import WebPage
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
@@ -49,7 +56,7 @@ class BaseFileImportMixin(TimeStampedModel):
         abstract = True
 
     def __str__(self):
-        return f"Scrobble import {self.id}"
+        return f"{self.import_type} import on {self.human_start}"
 
     @property
     def human_start(self):
@@ -60,14 +67,7 @@ class BaseFileImportMixin(TimeStampedModel):
 
     @property
     def import_type(self) -> str:
-        class_name = self.__class__.__name__
-        if class_name == "AudioscrobblerTSVImport":
-            return "Audioscrobbler"
-        if class_name == "KoReaderImport":
-            return "KoReader"
-        if self.__class__.__name__ == "LastFMImport":
-            return "LastFM"
-        return "Generic"
+        return "Unknown Import Source"
 
     def process(self, force=False):
         logger.warning("Process not implemented")
@@ -105,6 +105,15 @@ class BaseFileImportMixin(TimeStampedModel):
             ]
         )
 
+    def scrobbles(self) -> models.QuerySet:
+        scrobble_ids = []
+        if self.process_log:
+            for line in self.process_log.split("\n"):
+                sid = line.split("\t")[0]
+                if sid:
+                    scrobble_ids.append(sid)
+        return Scrobble.objects.filter(id__in=scrobble_ids)
+
     def mark_started(self):
         self.processing_started = timezone.now()
         self.save(update_fields=["processing_started"])
@@ -138,8 +147,9 @@ class KoReaderImport(BaseFileImportMixin):
     class Meta:
         verbose_name = "KOReader Import"
 
-    def __str__(self):
-        return f"KoReader import on {self.human_start}"
+    @property
+    def import_type(self) -> str:
+        return "KOReader"
 
     def get_absolute_url(self):
         return reverse(
@@ -182,8 +192,9 @@ class AudioScrobblerTSVImport(BaseFileImportMixin):
     class Meta:
         verbose_name = "AudioScrobbler TSV Import"
 
-    def __str__(self):
-        return f"Audioscrobbler import on {self.human_start}"
+    @property
+    def import_type(self) -> str:
+        return "AudiosScrobbler"
 
     def get_absolute_url(self):
         return reverse(
@@ -232,8 +243,9 @@ class LastFmImport(BaseFileImportMixin):
     class Meta:
         verbose_name = "Last.FM Import"
 
-    def __str__(self):
-        return f"LastFM import on {self.human_start}"
+    @property
+    def import_type(self) -> str:
+        return "LastFM"
 
     def get_absolute_url(self):
         return reverse(
@@ -274,6 +286,46 @@ class LastFmImport(BaseFileImportMixin):
         self.mark_finished()
 
 
+class RetroarchImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "Retroarch Import"
+
+    @property
+    def import_type(self) -> str:
+        return "Retroarch"
+
+    def get_absolute_url(self):
+        return reverse(
+            "scrobbles:retroarch-import-detail", kwargs={"slug": self.uuid}
+        )
+
+    def process(self, import_all=False, force=False):
+        """Import scrobbles found on Retroarch"""
+        if self.processed_finished and not force:
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
+            return
+
+        if force:
+            logger.info(f"You told me to force import from Retroarch")
+
+        if not self.user.profile.retroarch_path:
+            logger.info(
+                "Tying to import Retroarch logs, but user has no retroarch_path configured"
+            )
+
+        self.mark_started()
+
+        scrobbles = retroarch.import_retroarch_lrtl_files(
+            self.user.profile.retroarch_path,
+            self.user.id,
+        )
+
+        self.record_log(scrobbles)
+        self.mark_finished()
+
+
 class ChartRecord(TimeStampedModel):
     """Sort of like a materialized view for what we could dynamically generate,
     but would kill the DB as it gets larger. Collects time-based records
@@ -411,6 +463,19 @@ class ChartRecord(TimeStampedModel):
 class Scrobble(TimeStampedModel):
     """A scrobble tracks played media items by a user."""
 
+    class MediaType(models.TextChoices):
+        """Enum mapping a media model type to a string"""
+
+        VIDEO = "Video", "Video"
+        TRACK = "Track", "Track"
+        PODCAST_EPISODE = "Episode", "Podcast episode"
+        SPORT_EVENT = "SportEvent", "Sport event"
+        BOOK = "Book", "Book"
+        VIDEO_GAME = "VideoGame", "Video game"
+        BOARD_GAME = "BoardGame", "Board game"
+        GEO_LOCATION = "GeoLocation", "GeoLocation"
+        WEBPAGE = "WebPage", "Web Page"
+
     uuid = models.UUIDField(editable=False, **BNULL)
     video = models.ForeignKey(Video, on_delete=models.DO_NOTHING, **BNULL)
     track = models.ForeignKey(Track, on_delete=models.DO_NOTHING, **BNULL)
@@ -424,6 +489,16 @@ class Scrobble(TimeStampedModel):
     video_game = models.ForeignKey(
         VideoGame, on_delete=models.DO_NOTHING, **BNULL
     )
+    board_game = models.ForeignKey(
+        BoardGame, on_delete=models.DO_NOTHING, **BNULL
+    )
+    geo_location = models.ForeignKey(
+        GeoLocation, on_delete=models.DO_NOTHING, **BNULL
+    )
+    webpage = models.ForeignKey(WebPage, on_delete=models.DO_NOTHING, **BNULL)
+    media_type = models.CharField(
+        max_length=14, choices=MediaType.choices, default=MediaType.VIDEO
+    )
     user = models.ForeignKey(
         User, blank=True, null=True, on_delete=models.DO_NOTHING
     )
@@ -450,6 +525,21 @@ class Scrobble(TimeStampedModel):
     videogame_save_data = models.FileField(
         upload_to="scrobbles/videogame_save_data/", **BNULL
     )
+    videogame_screenshot = models.ImageField(
+        upload_to="scrobbles/videogame_screenshot/", **BNULL
+    )
+    videogame_screenshot_small = ImageSpecField(
+        source="videogame_screenshot",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    videogame_screenshot_medium = ImageSpecField(
+        source="videogame_screenshot",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
     long_play_seconds = models.BigIntegerField(**BNULL)
     long_play_complete = models.BooleanField(**BNULL)
 
@@ -457,6 +547,8 @@ class Scrobble(TimeStampedModel):
         if not self.uuid:
             self.uuid = uuid4()
 
+        self.media_type = self.MediaType(self.media_obj.__class__.__name__)
+
         return super(Scrobble, self).save(*args, **kwargs)
 
     @property
@@ -495,6 +587,40 @@ class Scrobble(TimeStampedModel):
             .first()
         )
 
+    @property
+    def previous_all(self) -> "Scrobble":
+        return (
+            Scrobble.objects.filter(media_type=self.media_type)
+            .order_by("-timestamp")
+            .filter(timestamp__lt=self.timestamp)
+            .first()
+        )
+
+    @property
+    def next_all(self) -> "Scrobble":
+        return (
+            Scrobble.objects.filter(media_type=self.media_type)
+            .order_by("-timestamp")
+            .filter(timestamp__gt=self.timestamp)
+            .first()
+        )
+
+    @property
+    def previous_all_media(self) -> "Scrobble":
+        return (
+            Scrobble.objects.order_by("-timestamp")
+            .filter(timestamp__lt=self.timestamp)
+            .first()
+        )
+
+    @property
+    def next_all_media(self) -> "Scrobble":
+        return (
+            Scrobble.objects.order_by("-timestamp")
+            .filter(timestamp__gt=self.timestamp)
+            .first()
+        )
+
     @property
     def session_pages_read(self) -> Optional[int]:
         """Look one scrobble back, if it isn't complete,"""
@@ -542,6 +668,7 @@ class Scrobble(TimeStampedModel):
     @property
     def can_be_updated(self) -> bool:
         updatable = True
+
         if self.media_obj.__class__.__name__ in LONG_PLAY_MEDIA.values():
             logger.info(f"No - Long play media")
             updatable = False
@@ -551,6 +678,37 @@ class Scrobble(TimeStampedModel):
         if self.is_stale:
             logger.info(f"No - stale - {self.id} - {self.source}")
             updatable = False
+        if self.media_obj.__class__.__name__ in ["GeoLocation"]:
+            if self.previous_all:
+                last_location = self.previous_all.media_obj
+                logger.info(
+                    f"Calculate proximity to last scrobble: {last_location}"
+                )
+
+                same_lat = last_location.lat == self.media_obj.lat
+                same_lon = last_location.lon == self.media_obj.lon
+                same_title = False
+                if self.media_obj.title:
+                    same_title = last_location.title == self.media_obj.title
+
+                logger.info(f"{self.timestamp}")
+                logger.info(f"Last lat: {last_location.lat}, last long {last_location.lon}, last title {last_location.title}")
+                logger.info(
+                    f"Our lat {self.media_obj.lat}, Our lon {self.media_obj.lon} or our title {self.media_obj.title}"
+                )
+
+                if (same_lat and same_lon) or same_title:
+                    logger.info(
+                        f"Yes - We're in the same place: {self.media_obj}"
+                    )
+                    updatable = True
+                else:
+                    logger.info(
+                        f"No - We've moved, start a new scrobble: {self.media_obj}"
+                    )
+                    # Stop the previous location scrobble
+                    self.previous_all.stop()
+                    updatable = False
         return updatable
 
     @property
@@ -568,6 +726,12 @@ class Scrobble(TimeStampedModel):
             media_obj = self.book
         if self.video_game:
             media_obj = self.video_game
+        if self.board_game:
+            media_obj = self.board_game
+        if self.geo_location:
+            media_obj = self.geo_location
+        if self.webpage:
+            media_obj = self.webpage
         return media_obj
 
     def __str__(self):
@@ -576,10 +740,11 @@ class Scrobble(TimeStampedModel):
 
     @classmethod
     def create_or_update(
-        cls, media, user_id: int, scrobble_data: dict
+        cls, media, user_id: int, scrobble_data: dict, **kwargs
     ) -> "Scrobble":
 
         media_class = media.__class__.__name__
+        dup = None
 
         media_query = models.Q(track=media)
         if media_class == "Track":
@@ -599,23 +764,41 @@ class Scrobble(TimeStampedModel):
         if media_class == "VideoGame":
             media_query = models.Q(video_game=media)
             scrobble_data["video_game_id"] = media.id
+        if media_class == "BoardGame":
+            media_query = models.Q(board_game=media)
+            scrobble_data["board_game_id"] = media.id
+        if media_class == "WebPage":
+            media_query = models.Q(webpage=media)
+            scrobble_data["webpage_id"] = media.id
+        if media_class == "GeoLocation":
+            media_query = models.Q(media_type=Scrobble.MediaType.GEO_LOCATION)
+            scrobble_data["geo_location_id"] = media.id
+            dup = cls.objects.filter(
+                media_type=cls.MediaType.GEO_LOCATION,
+                timestamp=scrobble_data.get("timestamp"),
+            ).first()
+
+            if dup:
+                logger.info(
+                    "[scrobbling] scrobble for geo location with identical timestamp found"
+                )
+                # TODO Fix return type, can we ever return a Scrobble?
+                return
 
         scrobble = (
             cls.objects.filter(
                 media_query,
                 user_id=user_id,
             )
-            .order_by("-modified")
+            .order_by("-timestamp")
             .first()
         )
-        # We want to finish a video after it's reeached 90%
-        # but we also don't want to create new videos, so in
-        # this special case, we allow jellyfin resumed status
-        # to allow a scrobble to be updated.
-        jellyfin_in_progress = scrobble_data.get("jellyfin_status", None)
-        if scrobble and (scrobble.can_be_updated or jellyfin_in_progress):
+
+        if scrobble and scrobble.can_be_updated:
+            source = scrobble_data["source"]
+            mtype = media.__class__.__name__
             logger.info(
-                f"Updating {scrobble.id}",
+                f"[scrobbling] updating {scrobble.id} for {mtype} {media.id} from {source}",
                 {"scrobble_data": scrobble_data, "media": media},
             )
             return scrobble.update(scrobble_data)
@@ -624,9 +807,9 @@ class Scrobble(TimeStampedModel):
         scrobble_data.pop("mopidy_status", None)
         scrobble_data.pop("jellyfin_status", None)
         source = scrobble_data["source"]
+        mtype = media.__class__.__name__
         logger.info(
-            f"Creating for {media.id} - {source}",
-            {"scrobble_data": scrobble_data, "media": media},
+            f"[scrobbling] creating for {mtype} {media.id} from {source}"
         )
         return cls.create(scrobble_data)
 
@@ -656,6 +839,7 @@ class Scrobble(TimeStampedModel):
                 scrobble_data.pop("timestamp", None) or timezone.now()
             )
 
+        scrobble_data.pop("timestamp", None)
         update_fields = []
         for key, value in scrobble_data.items():
             setattr(self, key, value)
@@ -676,14 +860,28 @@ class Scrobble(TimeStampedModel):
         return scrobble
 
     def stop(self, force_finish=False) -> None:
+        self.stop_timestamp = timezone.now()
+        if force_finish:
+            self.played_to_completion = True
         self.in_progress = False
-        self.save(update_fields=["in_progress"])
+        if not self.playback_position_seconds:
+            self.playback_position_seconds = int(
+                (self.stop_timestamp - self.timestamp).total_seconds()
+            )
+
+        self.save(
+            update_fields=[
+                "in_progress",
+                "played_to_completion",
+                "stop_timestamp",
+                "playback_position_seconds",
+            ]
+        )
         logger.info(f"stopping {self.id} from {self.source}")
 
         class_name = self.media_obj.__class__.__name__
         if class_name in LONG_PLAY_MEDIA.values():
             check_long_play_for_finish(self)
-            return
 
     def pause(self) -> None:
         if self.is_paused:
@@ -710,3 +908,96 @@ class Scrobble(TimeStampedModel):
             f"{self.id} - {self.playback_position_seconds} - {self.source}"
         )
         self.save(update_fields=["playback_position_seconds"])
+
+
+class ScrobbledPage(TimeStampedModel):
+    scrobble = models.ForeignKey(Scrobble, on_delete=models.DO_NOTHING)
+    number = models.IntegerField()
+    start_time = models.DateTimeField(**BNULL)
+    end_time = models.DateTimeField(**BNULL)
+    duration_seconds = models.IntegerField(**BNULL)
+    notes = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return f"Page {self.number} of {self.book.pages} in {self.book.title}"
+
+    def save(self, *args, **kwargs):
+        if not self.end_time and self.duration_seconds:
+            self._set_end_time()
+
+        return super(ScrobbledPage, self).save(*args, **kwargs)
+
+    @cached_property
+    def book(self):
+        return self.scrobble.book
+
+    @property
+    def next(self):
+        user_pages_qs = self.book.scrobbledpage_set.filter(
+            user=self.scrobble.user
+        )
+        page = user_pages_qs.filter(number=self.number + 1).first()
+        if not page:
+            page = (
+                user_pages_qs.filter(created__gt=self.created)
+                .order_by("created")
+                .first()
+            )
+        return page
+
+    @property
+    def previous(self):
+        user_pages_qs = self.book.scrobbledpage_set.filter(
+            user=self.scrobble.user
+        )
+        page = user_pages_qs.filter(number=self.number - 1).first()
+        if not page:
+            page = (
+                user_pages_qs.filter(created__lt=self.created)
+                .order_by("-created")
+                .first()
+            )
+        return page
+
+    @property
+    def seconds_to_next_page(self) -> int:
+        seconds = 999999  # Effectively infnity time as we have no next
+        if not self.end_time:
+            self._set_end_time()
+        if self.next:
+            seconds = (self.next.start_time - self.end_time).seconds
+        return seconds
+
+    @property
+    def is_scrobblable(self) -> bool:
+        """A page defines the start of a scrobble if the seconds to next page
+        are greater than an hour, or 3600 seconds, and it's not a single page,
+        so the next seconds to next_page is less than an hour as well.
+
+        As a special case, the first recorded page is a scrobble, so we establish
+        when the book was started.
+
+        """
+        is_scrobblable = False
+        over_an_hour_since_last_page = False
+        if not self.previous:
+            is_scrobblable = True
+
+        if self.previous:
+            over_an_hour_since_last_page = (
+                self.previous.seconds_to_next_page >= 3600
+            )
+        blip = self.seconds_to_next_page >= 3600
+
+        if over_an_hour_since_last_page and not blip:
+            is_scrobblable = True
+        return is_scrobblable
+
+    def _set_end_time(self) -> None:
+        if self.end_time:
+            return
+
+        self.end_time = self.start_time + datetime.timedelta(
+            seconds=self.duration_seconds
+        )
+        self.save(update_fields=["end_time"])

vrobbler/apps/scrobbles/scrobblers.py

@@ -1,8 +1,15 @@
 import logging
 from typing import Optional
 
+import pendulum
+from boardgames.bgg import lookup_boardgame_from_bgg
+from boardgames.models import BoardGame
+from books.models import Book
+from books.openlibrary import lookup_book_from_openlibrary
 from dateutil.parser import parse
 from django.utils import timezone
+from locations.constants import LOCATION_PROVIDERS
+from locations.models import GeoLocation
 from music.constants import JELLYFIN_POST_KEYS
 from music.models import Track
 from music.utils import (
@@ -14,9 +21,11 @@ from podcasts.models import Episode
 from scrobbles.models import Scrobble
 from scrobbles.utils import convert_to_seconds, parse_mopidy_uri
 from sports.models import SportEvent
-from videos.models import Video
+from sports.thesportsdb import lookup_event_from_thesportsdb
+from videogames.howlongtobeat import lookup_game_from_hltb
 from videogames.models import VideoGame
-from books.models import Book
+from videos.models import Video
+from webpages.models import WebPage
 
 logger = logging.getLogger(__name__)
 
@@ -169,8 +178,9 @@ def manual_scrobble_video(imdb_id: str, user_id: int):
     video = Video.find_or_create({"imdb_id": imdb_id})
 
     # When manually scrobbling, try finding a source from the series
+    source = "Vrobbler"
     if video.tv_series:
-        source = video.tv_series.preferred_source or "Vrobbler"
+        source = video.tv_series.preferred_source
     scrobble_dict = {
         "user_id": user_id,
         "timestamp": timezone.now(),
@@ -182,15 +192,16 @@ def manual_scrobble_video(imdb_id: str, user_id: int):
     return Scrobble.create_or_update(video, user_id, scrobble_dict)
 
 
-def manual_scrobble_event(data_dict: dict, user_id: Optional[int]):
+def manual_scrobble_event(thesportsdb_id: str, user_id: int):
+    data_dict = lookup_event_from_thesportsdb(thesportsdb_id)
+
     event = SportEvent.find_or_create(data_dict)
-
     scrobble_dict = build_scrobble_dict(data_dict, user_id)
-
     return Scrobble.create_or_update(event, user_id, scrobble_dict)
 
 
-def manual_scrobble_video_game(data_dict: dict, user_id: Optional[int]):
+def manual_scrobble_video_game(hltb_id: str, user_id: int):
+    data_dict = lookup_game_from_hltb(hltb_id)
     game = VideoGame.find_or_create(data_dict)
 
     scrobble_dict = {
@@ -205,8 +216,8 @@ def manual_scrobble_video_game(data_dict: dict, user_id: Optional[int]):
     return Scrobble.create_or_update(game, user_id, scrobble_dict)
 
 
-def manual_scrobble_book(data_dict: dict, user_id: Optional[int]):
-    book = Book.find_or_create(data_dict)
+def manual_scrobble_book(openlibrary_id: str, user_id: int):
+    book = Book.find_or_create(openlibrary_id)
 
     scrobble_dict = {
         "user_id": user_id,
@@ -217,3 +228,66 @@ def manual_scrobble_book(data_dict: dict, user_id: Optional[int]):
     }
 
     return Scrobble.create_or_update(book, user_id, scrobble_dict)
+
+
+def manual_scrobble_board_game(bggeek_id: str, user_id: int):
+    boardgame = BoardGame.find_or_create(bggeek_id)
+
+    if not boardgame:
+        logger.error(f"No board game found for ID {bggeek_id}")
+        return
+
+    scrobble_dict = {
+        "user_id": user_id,
+        "timestamp": timezone.now(),
+        "playback_position_seconds": 0,
+        "source": "Vrobbler",
+        "source_id": "Manually scrobbled from Vrobbler and looked up via boardgamegeek.com",
+    }
+
+    return Scrobble.create_or_update(boardgame, user_id, scrobble_dict)
+
+
+def manual_scrobble_webpage(url: str, user_id: int):
+    webpage = WebPage.find_or_create({"url": url})
+
+    scrobble_dict = {
+        "user_id": user_id,
+        "timestamp": timezone.now(),
+        "playback_position_seconds": 0,
+        "source": "Vrobbler",
+        "source_id": "Manually scrobbled from Vrobbler",
+    }
+
+    return Scrobble.create_or_update(webpage, user_id, scrobble_dict)
+
+
+def gpslogger_scrobble_location(
+    data_dict: dict, user_id: Optional[int]
+) -> Optional[Scrobble]:
+    # Save the data coming in
+    if not user_id:
+        user_id = 1  # TODO fix authing the end point to get user
+
+    location = GeoLocation.find_or_create(data_dict)
+
+    # Now we run off a scrobble
+    playback_seconds = 1
+    extra_data = {
+        "user_id": user_id,
+        "timestamp": pendulum.parse(data_dict.get("time", timezone.now())),
+        "playback_position_seconds": playback_seconds,
+        "source": "GPSLogger",
+    }
+
+    scrobble = Scrobble.create_or_update(location, user_id, extra_data)
+
+    provider = f"data source: {LOCATION_PROVIDERS[data_dict.get('prov')]}"
+    if scrobble:
+        if scrobble.notes:
+            scrobble.notes = scrobble.notes + f"\n{provider}"
+        else:
+            scrobble.notes = provider
+        scrobble.save(update_fields=["notes"])
+
+    return scrobble

vrobbler/apps/scrobbles/stats.py

@@ -47,6 +47,10 @@ def get_scrobble_count_qs(
         data_model = apps.get_model(
             app_label="sports", model_name="SportEvent"
         )
+    if model_str == "GeoLocation":
+        data_model = apps.get_model(
+            app_label="locations", model_name="GeoLocation"
+        )
 
     if model_str == "Artist":
         base_qs = data_model.objects.filter(

vrobbler/apps/scrobbles/tasks.py

@@ -1,20 +1,27 @@
 import logging
 
 from celery import shared_task
+from django.apps import apps
 from django.contrib.auth import get_user_model
-from scrobbles.models import (
-    AudioScrobblerTSVImport,
-    KoReaderImport,
-    LastFmImport,
-)
 from scrobbles.stats import build_yesterdays_charts_for_user
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
 
 
+@shared_task
+def process_retroarch_import(import_id):
+    RetroarchImport = apps.get_model("scrobbles", "RetroarchImport")
+    retroarch_import = RetroarchImport.objects.filter(id=import_id).first()
+    if not retroarch_import:
+        logger.warn(f"RetroarchImport not found with id {import_id}")
+
+    retroarch_import.process()
+
+
 @shared_task
 def process_lastfm_import(import_id):
+    LastFmImport = apps.get_model("scrobbles", "LastFMImport")
     lastfm_import = LastFmImport.objects.filter(id=import_id).first()
     if not lastfm_import:
         logger.warn(f"LastFmImport not found with id {import_id}")
@@ -24,6 +31,9 @@ def process_lastfm_import(import_id):
 
 @shared_task
 def process_tsv_import(import_id):
+    AudioScrobblerTSVImport = apps.get_model(
+        "scrobbles", "AudioscrobblerTSVImport"
+    )
     tsv_import = AudioScrobblerTSVImport.objects.filter(id=import_id).first()
     if not tsv_import:
         logger.warn(f"AudioScrobblerTSVImport not found with id {import_id}")
@@ -33,6 +43,7 @@ def process_tsv_import(import_id):
 
 @shared_task
 def process_koreader_import(import_id):
+    KoReaderImport = apps.get_model("scrobbles", "KoReaderImport")
     koreader_import = KoReaderImport.objects.filter(id=import_id).first()
     if not koreader_import:
         logger.warn(f"KOReaderImport not found with id {import_id}")

vrobbler/apps/scrobbles/tsv.py

@@ -13,6 +13,8 @@ from music.utils import (
 from scrobbles.constants import AsTsvColumn
 from scrobbles.models import Scrobble
 
+from vrobbler.apps.scrobbles.utils import timestamp_user_tz_to_utc
+
 logger = logging.getLogger(__name__)
 
 
@@ -65,10 +67,8 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
             )
             continue
 
-        timestamp = (
-            datetime.utcfromtimestamp(int(row[AsTsvColumn["TIMESTAMP"].value]))
-            .replace(tzinfo=user_tz)
-            .astimezone(pytz.utc)
+        timestamp = timestamp_user_tz_to_utc(
+            int(row[AsTsvColumn["TIMESTAMP"].value]), user_tz
         )
 
         new_scrobble = Scrobble(
@@ -79,6 +79,7 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
             track=track,
             played_to_completion=True,
             in_progress=False,
+            media_type=Scrobble.MediaType.TRACK,
         )
         existing = Scrobble.objects.filter(
             timestamp=timestamp, track=track

vrobbler/apps/scrobbles/urls.py

@@ -34,6 +34,11 @@ urlpatterns = [
         views.lastfm_import,
         name="lastfm-import",
     ),
+    path(
+        "webhook/gps/",
+        views.gps_webhook,
+        name="gps-webhook",
+    ),
     path(
         "webhook/jellyfin/",
         views.jellyfin_webhook,
@@ -65,6 +70,11 @@ urlpatterns = [
         views.ScrobbleKoReaderImportDetailView.as_view(),
         name="koreader-import-detail",
     ),
+    path(
+        "imports/retroarch/<slug:slug>/",
+        views.ScrobbleRetroarchImportDetailView.as_view(),
+        name="retroarch-import-detail",
+    ),
     path(
         "charts/",
         views.ChartRecordView.as_view(),

vrobbler/apps/scrobbles/utils.py

@@ -1,20 +1,31 @@
 import logging
+import os
+from datetime import datetime, timedelta, tzinfo
 from urllib.parse import unquote
 
-from django.utils import timezone
+import pytz
 from dateutil.parser import ParserError, parse
 from django.apps import apps
-from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.db import models
-from vrobbler.apps.profiles.utils import now_user_timezone
-
-from vrobbler.apps.scrobbles.constants import LONG_PLAY_MEDIA
+from django.utils import timezone
+from profiles.models import UserProfile
+from profiles.utils import now_user_timezone
+from scrobbles.constants import LONG_PLAY_MEDIA
+from scrobbles.tasks import process_lastfm_import, process_retroarch_import
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
 
 
+PODCAST_DATE_FORMAT = "YYYY-MM-DD"
+
+def timestamp_user_tz_to_utc(timestamp: int, user_tz: tzinfo) -> datetime:
+    return user_tz.localize(datetime.utcfromtimestamp(timestamp)).astimezone(
+        pytz.utc
+    )
+
+
 def convert_to_seconds(run_time: str) -> int:
     """Jellyfin sends run time as 00:00:00 string. We want the run time to
     actually be in seconds so we'll convert it"
@@ -31,46 +42,40 @@ def convert_to_seconds(run_time: str) -> int:
         run_time_int = int((((hours * 60) + minutes) * 60) + seconds)
     return run_time_int
 
-
 def parse_mopidy_uri(uri: str) -> dict:
     logger.debug(f"Parsing URI: {uri}")
-    parsed_uri = uri.split("/")
+    parsed_uri = os.path.splitext(unquote(uri))[0].split("/")
 
-    episode_str = unquote(parsed_uri.pop(-1).strip(".mp3"))
-    podcast_str = unquote(parsed_uri.pop(-1))
-    possible_date_str = episode_str[0:10]
+    episode_str = parsed_uri[-1]
+    podcast_name = parsed_uri[-2].strip()
+    episode_num = None
+    episode_num_pad = 0
 
     try:
-        pub_date = parse(possible_date_str)
+        # Without episode numbers the date will lead
+        pub_date = parse(episode_str[0:10])
     except ParserError:
-        pub_date = ""
-    logger.debug(f"Found pub date {pub_date} from Mopidy URI")
+        episode_num = int(episode_str.split("-")[0])
+        episode_num_pad = len(str(episode_num)) + 1
 
-    try:
-        if pub_date:
-            episode_num = int(episode_str.split("-")[3])
-        else:
-            episode_num = int(episode_str.split("-")[0])
-    except IndexError:
-        episode_num = None
-    except ValueError:
-        episode_num = None
-    logger.debug(f"Found episode num {episode_num} from Mopidy URI")
+        try:
+            # Beacuse we have epsiode numbers on
+            pub_date = episode_str[episode_num_pad:14]
+        except ParserError:
+            pub_date = ""
 
+    gap_to_strip = 0
     if pub_date:
-        episode_str = episode_str.strip(episode_str[:11])
+        gap_to_strip += len(PODCAST_DATE_FORMAT)
+    if episode_num:
+        gap_to_strip += episode_num_pad
 
-    if type(episode_num) is int:
-        episode_num_gap = len(str(episode_num)) + 1
-        episode_str = episode_str.strip(episode_str[:episode_num_gap])
-
-    episode_str = episode_str.replace("-", " ")
-    logger.debug(f"Found episode name {episode_str} from Mopidy URI")
+    episode_name = episode_str[gap_to_strip:].replace("-", " ").strip()
 
     return {
-        "episode_filename": episode_str,
+        "episode_filename": episode_name,
         "episode_num": episode_num,
-        "podcast_name": podcast_str,
+        "podcast_name": podcast_name,
         "pub_date": pub_date,
     }
 
@@ -104,29 +109,18 @@ def check_scrobble_for_finish(
 
 
 def check_long_play_for_finish(scrobble):
-    Scrobble = apps.get_model("scrobbles", "Scrobble")
-    class_name = scrobble.media_obj.__class__.__name__
-    now = timezone.now()
+    seconds_elapsed = (timezone.now() - scrobble.timestamp).seconds
+    past_seconds = 0
+
+    # Set our playback seconds, and calc long play seconds
+    scrobble.playback_position_seconds = seconds_elapsed
+    if scrobble.previous:
+        past_seconds = scrobble.previous.long_play_seconds
+
+    scrobble.long_play_seconds = past_seconds + seconds_elapsed
+
+    # Long play scrobbles are always finished when we say they are
     scrobble.played_to_completion = True
-    scrobble.playback_position_seconds = (now - scrobble.timestamp).seconds
-
-    media_filter = models.Q(video_game=scrobble.video_game)
-    if class_name == "Book":
-        media_filter = models.Q(book=scrobble.book)
-
-    # Check for last scrobble, and if present, update long play time
-    last_scrobble = Scrobble.objects.filter(
-        media_filter,
-        user_id=scrobble.user,
-        played_to_completion=True,
-        long_play_complete=False,
-    ).last()
-    scrobble.long_play_seconds = scrobble.playback_position_seconds
-    if last_scrobble:
-        scrobble.long_play_seconds = (
-            last_scrobble.long_play_seconds
-            + scrobble.playback_position_seconds
-        )
 
     scrobble.save(
         update_fields=[
@@ -153,6 +147,10 @@ def get_scrobbles_for_media(media_obj, user: User) -> models.QuerySet:
     return Scrobble.objects.filter(media_query, user=user)
 
 
+def get_recently_played_board_games(user: User) -> dict:
+    ...
+
+
 def get_long_plays_in_progress(user: User) -> dict:
     """Find all books where the last scrobble is not marked complete"""
     media_dict = {
@@ -184,9 +182,85 @@ def get_long_plays_completed(user: User) -> list:
             if (
                 media.scrobble_set.all()
                 and media.scrobble_set.filter(user=user)
+                .order_by("timestamp")
                 .last()
                 .long_play_complete
                 == True
             ):
                 media_list.append(media)
     return media_list
+
+
+def import_lastfm_for_all_users(restart=False):
+    """Grab a list of all users with LastFM enabled and kickoff imports for them"""
+    LastFmImport = apps.get_model("scrobbles", "LastFMImport")
+    lastfm_enabled_user_ids = UserProfile.objects.filter(
+        lastfm_username__isnull=False,
+        lastfm_password__isnull=False,
+        lastfm_auto_import=True,
+    ).values_list("user_id", flat=True)
+
+    lastfm_import_count = 0
+
+    for user_id in lastfm_enabled_user_ids:
+        lfm_import, created = LastFmImport.objects.get_or_create(
+            user_id=user_id, processed_finished__isnull=True
+        )
+        if not created and not restart:
+            logger.info(
+                f"Not resuming failed LastFM import {lfm_import.id} for user {user_id}, use restart=True to restart"
+            )
+            continue
+        process_lastfm_import.delay(lfm_import.id)
+        lastfm_import_count += 1
+    return lastfm_import_count
+
+
+def import_retroarch_for_all_users(restart=False):
+    """Grab a list of all users with Retroarch enabled and kickoff imports for them"""
+    RetroarchImport = apps.get_model("scrobbles", "RetroarchImport")
+    retroarch_enabled_user_ids = UserProfile.objects.filter(
+        retroarch_path__isnull=False,
+        retroarch_auto_import=True,
+    ).values_list("user_id", flat=True)
+
+    retroarch_import_count = 0
+
+    for user_id in retroarch_enabled_user_ids:
+        retroarch_import, created = RetroarchImport.objects.get_or_create(
+            user_id=user_id, processed_finished__isnull=True
+        )
+        if not created and not restart:
+            logger.info(
+                f"Not resuming failed LastFM import {retroarch_import.id} for user {user_id}, use restart=True to restart"
+            )
+            continue
+        process_retroarch_import.delay(retroarch_import.id)
+        retroarch_import_count += 1
+    return retroarch_import_count
+
+
+def delete_zombie_scrobbles(dry_run=True):
+    """Look for any scrobble over a day old that is not paused and still in progress and delete it"""
+    Scrobble = apps.get_model("scrobbles", "Scrobble")
+    now = timezone.now()
+    three_days_ago = now - timedelta(days=3)
+
+    # TODO This should be part of a custom manager
+    zombie_scrobbles = Scrobble.objects.filter(
+        timestamp__lte=three_days_ago,
+        is_paused=False,
+        played_to_completion=False,
+    )
+
+    zombies_found = zombie_scrobbles.count()
+
+    if not dry_run:
+        logger.info(f"Deleted {zombies_found} zombie scrobbles")
+        zombie_scrobbles.delete()
+        return zombies_found
+
+    logger.info(
+        f"Found {zombies_found} zombie scrobbles to delete, use dry_run=False to proceed"
+    )
+    return zombies_found

vrobbler/apps/scrobbles/views.py

@@ -9,10 +9,9 @@ from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import LoginRequiredMixin
 from django.db.models import Q
-from django.db.models.fields import timezone
 from django.db.models.query import QuerySet
 from django.http import FileResponse, HttpResponseRedirect, JsonResponse
-from django.urls import reverse, reverse_lazy
+from django.urls import reverse_lazy
 from django.utils import timezone
 from django.views.decorators.csrf import csrf_exempt
 from django.views.generic import DetailView, FormView, TemplateView
@@ -33,6 +32,8 @@ from scrobbles.constants import (
     JELLYFIN_AUDIO_ITEM_TYPES,
     JELLYFIN_VIDEO_ITEM_TYPES,
     LONG_PLAY_MEDIA,
+    MANUAL_SCROBBLE_FNS,
+    PLAY_AGAIN_MEDIA,
 )
 from scrobbles.export import export_scrobbles
 from scrobbles.forms import ExportScrobbleForm, ScrobbleForm
@@ -41,30 +42,19 @@ from scrobbles.models import (
     ChartRecord,
     KoReaderImport,
     LastFmImport,
+    RetroarchImport,
     Scrobble,
 )
-from scrobbles.scrobblers import (
-    jellyfin_scrobble_track,
-    jellyfin_scrobble_video,
-    manual_scrobble_book,
-    manual_scrobble_event,
-    manual_scrobble_video,
-    manual_scrobble_video_game,
-    mopidy_scrobble_podcast,
-    mopidy_scrobble_track,
-)
+from scrobbles.scrobblers import *
 from scrobbles.tasks import (
     process_koreader_import,
     process_lastfm_import,
     process_tsv_import,
 )
-from sports.thesportsdb import lookup_event_from_thesportsdb
-from videogames.howlongtobeat import lookup_game_from_hltb
-
-from books.openlibrary import lookup_book_from_openlibrary
 from scrobbles.utils import (
     get_long_plays_completed,
     get_long_plays_in_progress,
+    get_recently_played_board_games,
 )
 
 logger = logging.getLogger(__name__)
@@ -81,6 +71,7 @@ class RecentScrobbleList(ListView):
                 played_to_completion=True, user=user
             )
             data["long_play_in_progress"] = get_long_plays_in_progress(user)
+            data["play_again"] = get_recently_played_board_games(user)
             data["video_scrobble_list"] = completed_for_user.filter(
                 video__isnull=False
             ).order_by("-timestamp")[:15]
@@ -93,6 +84,14 @@ class RecentScrobbleList(ListView):
                 sport_event__isnull=False
             ).order_by("-timestamp")[:15]
 
+            data["videogame_scrobble_list"] = completed_for_user.filter(
+                video_game__isnull=False
+            ).order_by("-timestamp")[:15]
+
+            data["boardgame_scrobble_list"] = completed_for_user.filter(
+                board_game__isnull=False
+            ).order_by("-timestamp")[:15]
+
             data["active_imports"] = AudioScrobblerTSVImport.objects.filter(
                 processing_started__isnull=False,
                 processed_finished__isnull=True,
@@ -162,13 +161,16 @@ class ScrobbleImportListView(TemplateView):
 
         context_data["tsv_imports"] = AudioScrobblerTSVImport.objects.filter(
             user=self.request.user,
-        ).order_by("-processing_started")
+        ).order_by("-processing_started")[:10]
         context_data["koreader_imports"] = KoReaderImport.objects.filter(
             user=self.request.user,
-        ).order_by("-processing_started")
+        ).order_by("-processing_started")[:10]
         context_data["lastfm_imports"] = LastFmImport.objects.filter(
             user=self.request.user,
-        ).order_by("-processing_started")
+        ).order_by("-processing_started")[:10]
+        context_data["retroarch_imports"] = RetroarchImport.objects.filter(
+            user=self.request.user,
+        ).order_by("-processing_started")[:10]
         return context_data
 
 
@@ -188,6 +190,8 @@ class BaseScrobbleImportDetailView(DetailView):
             title = "Audioscrobbler TSV Import"
         if self.model == LastFmImport:
             title = "LastFM Import"
+        if self.model == RetroarchImport:
+            title = "Retroarch Import"
         context_data["title"] = title
         return context_data
 
@@ -204,37 +208,25 @@ class ScrobbleLastFMImportDetailView(BaseScrobbleImportDetailView):
     model = LastFmImport
 
 
+class ScrobbleRetroarchImportDetailView(BaseScrobbleImportDetailView):
+    model = RetroarchImport
+
+
 class ManualScrobbleView(FormView):
     form_class = ScrobbleForm
     template_name = "scrobbles/manual_form.html"
 
     def form_valid(self, form):
-
         item_str = form.cleaned_data.get("item_id")
-        key = item_str[:2]
-        item_id = item_str[3:]
-        data_dict = None
+        logger.debug(f"Looking for scrobblable media with input {item_str}")
 
-        if key == "-v":
-            logger.debug(f"Looking for video game with ID {item_id}")
-            data_dict = lookup_game_from_hltb(item_id.replace("-v", ""))
-            if data_dict:
-                manual_scrobble_video_game(data_dict, self.request.user.id)
+        key, item_id = item_str[:2], item_str[3:]
+        scrobble_fn = MANUAL_SCROBBLE_FNS[key]
+        scrobble = eval(scrobble_fn)(item_id, self.request.user.id)
 
-        if key == "-b":
-            logger.debug(f"Looking for book with ID {item_id}")
-            data_dict = lookup_book_from_openlibrary(item_id.replace("-b", ""))
-            if data_dict:
-                manual_scrobble_book(data_dict, self.request.user.id)
-
-        if key == "-s":
-            logger.debug(f"Looking for sport event with ID {item_id}")
-            data_dict = lookup_event_from_thesportsdb(item_id)
-            if data_dict:
-                manual_scrobble_event(data_dict, self.request.user.id)
-
-        if key == "-i":
-            manual_scrobble_video(item_id, self.request.user.id)
+        if self.request.user.profile.redirect_to_webpage and key == "-w":
+            logger.info(f"Redirecting to {scrobble.media_obj} detail page")
+            return HttpResponseRedirect(scrobble.media_obj.get_absolute_url())
 
         return HttpResponseRedirect(self.request.META.get("HTTP_REFERER"))
 
@@ -316,16 +308,18 @@ def lastfm_import(request):
 def jellyfin_webhook(request):
     data_dict = request.data
 
-    if (
-        data_dict["NotificationType"] == "PlaybackProgress"
-        and data_dict["ItemType"] == "Audio"
-    ):
-        return Response({}, status=status.HTTP_304_NOT_MODIFIED)
-
     # For making things easier to build new input processors
     if getattr(settings, "DUMP_REQUEST_DATA", False):
         json_data = json.dumps(data_dict, indent=4)
-        logger.debug(f"{json_data}")
+        logger.info(f"{json_data}")
+
+    in_progress = data_dict.get("NotificationType", "") == "PlaybackProgress"
+    is_music = data_dict.get("ItemType", "") == "Audio"
+
+    # Disregard progress updates
+    if in_progress and is_music:
+        logger.debug("Disregarding playback update from Jellyfin")
+        return Response({}, status=status.HTTP_304_NOT_MODIFIED)
 
     scrobble = None
     media_type = data_dict.get("ItemType", "")
@@ -349,7 +343,6 @@ def mopidy_webhook(request):
     try:
         data_dict = json.loads(request.data)
     except TypeError:
-        logger.warning("Received Mopidy data as dict, rather than a string")
         data_dict = request.data
 
     # For making things easier to build new input processors
@@ -368,6 +361,28 @@ def mopidy_webhook(request):
     return Response({"scrobble_id": scrobble.id}, status=status.HTTP_200_OK)
 
 
+@csrf_exempt
+@permission_classes([IsAuthenticated])
+@api_view(["POST"])
+def gps_webhook(request):
+    try:
+        data_dict = json.loads(request.data)
+    except TypeError:
+        data_dict = request.data
+
+    # For making things easier to build new input processors
+    if getattr(settings, "DUMP_REQUEST_DATA", False):
+        json_data = json.dumps(data_dict, indent=4)
+        logger.debug(f"{json_data}")
+
+    scrobble = gpslogger_scrobble_location(data_dict, request.user.id)
+
+    if not scrobble:
+        return Response({}, status=status.HTTP_200_OK)
+
+    return Response({"scrobble_id": scrobble.id}, status=status.HTTP_200_OK)
+
+
 @csrf_exempt
 @permission_classes([IsAuthenticated])
 @api_view(["POST"])
@@ -400,11 +415,10 @@ def scrobble_start(request, uuid):
     success_url = request.META.get("HTTP_REFERER")
 
     if not user.is_authenticated:
-
         return HttpResponseRedirect(success_url)
 
     media_obj = None
-    for app, model in LONG_PLAY_MEDIA.items():
+    for app, model in PLAY_AGAIN_MEDIA.items():
         media_model = apps.get_model(app_label=app, model_name=model)
         media_obj = media_model.objects.filter(uuid=uuid).first()
         if media_obj:
@@ -415,15 +429,15 @@ def scrobble_start(request, uuid):
 
     scrobble = None
     user_id = request.user.id
-    if media_obj and media_obj.__class__.__name__ == "Book":
-        data_dict = {
-            "title": media_obj.title,
-            "author": media_obj.author.name,
-        }
-        scrobble = manual_scrobble_book(data_dict, user_id)
-    if media_obj and media_obj.__class__.__name__ == "VideoGame":
-        data_dict = {"hltb_id": media_obj.hltb_id}
-        scrobble = manual_scrobble_video_game(data_dict, user_id)
+    if media_obj:
+        if media_obj.__class__.__name__ == Scrobble.MediaType.BOOK:
+            scrobble = manual_scrobble_book(media_obj.openlibrary_id, user_id)
+        if media_obj.__class__.__name__ == Scrobble.MediaType.VIDEO_GAME:
+            scrobble = manual_scrobble_video_game(media_obj.hltb_id, user_id)
+        if media_obj.__class__.__name__ == Scrobble.MediaType.BOARD_GAME:
+            scrobble = manual_scrobble_board_game(media_obj.bggeek_id, user_id)
+        if media_obj.__class__.__name__ == Scrobble.MediaType.WEBPAGE:
+            scrobble = manual_scrobble_webpage(media_obj.url, user_id)
 
     if scrobble:
         messages.add_message(
@@ -435,6 +449,13 @@ def scrobble_start(request, uuid):
         messages.add_message(
             request, messages.ERROR, f"Media with uuid {uuid} not found."
         )
+
+    if (
+        user.profile.redirect_on_webpage
+        and media_obj.__class__.__name__ == Scrobble.MediaType.WEBPAGE
+    ):
+        logger.info(f"Redirecting to {media_obj} detail apge")
+        return HttpResponseRedirect(media_obj.get_absolute_url())
     return HttpResponseRedirect(success_url)
 
 

vrobbler/apps/sports/migrations/0014_alter_sportevent_genre.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-04-18 02:33
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+        ("sports", "0013_sportevent_genre"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="sportevent",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                blank=True,
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/sports/models.py

@@ -238,9 +238,9 @@ class SportEvent(ScrobblableMixin):
             "away_team": away_team,
             "player_one": player_one,
             "player_two": player_two,
-            "start": data_dict["Start"],
+            "start": data_dict.get("Start"),
             "round": round,
-            "run_time_seconds": data_dict.get("RunTime", 0),
+            "run_time_seconds": data_dict.get("RunTime"),
         }
         event, _created = cls.objects.get_or_create(**event_dict)
 

vrobbler/apps/sports/thesportsdb.py

@@ -21,18 +21,21 @@ def lookup_event_from_thesportsdb(event_id: str) -> dict:
 
     if not event or type(event) != dict:
         return {}
-    league = {}  # client.lookup_league(league_id=event.get('idLeague'))
-    event_type = "Game"
     sport, _created = Sport.objects.get_or_create(
         thesportsdb_id=event.get("strSport")
     )
 
+    try:
+        start = parse(event.get("strTimestamp"))
+    except:
+        start = timezone.now()
+
     data_dict = {
         "EventId": event_id,
         "ItemType": sport.default_event_type,
         "Name": event.get("strEvent"),
         "AltName": event.get("strEventAlternate"),
-        "Start": parse(event.get("strTimestamp")),
+        "Start": start,
         "Provider_thesportsdb": event.get("idEvent"),
         "RunTime": sport.default_event_run_time_seconds,
         "Sport": event.get("strSport"),

vrobbler/apps/videogames/exceptions.py

@@ -0,0 +1,2 @@
+class GameNotFound(Exception):
+    pass

vrobbler/apps/videogames/igdb.py

@@ -42,6 +42,8 @@ def lookup_game_id_from_gdb(name: str) -> str:
         "Authorization": f"Bearer {get_igdb_token()}",
         "Client-ID": IGDB_CLIENT_ID,
     }
+    if "(" in name:
+        name = name.split(" (")[0]
 
     body = f'fields name,game,published_at; search "{name}"; limit 20;'
     response = requests.post(SEARCH_URL, data=body, headers=headers)
@@ -53,11 +55,16 @@ def lookup_game_id_from_gdb(name: str) -> str:
     return results[0]["game"]
 
 
-def lookup_game_from_igdb(igdb_id: str) -> Dict:
+def lookup_game_from_igdb(name_or_igdb_id: str) -> Dict:
     """Given credsa and an IGDB game ID, lookup the game metadata and return it
     in a dictionary mapped to our internal game fields
 
     """
+    try:
+        igdb_id = int(name_or_igdb_id)
+    except ValueError:
+        igdb_id = lookup_game_id_from_gdb(name_or_igdb_id)
+
     headers = {
         "Authorization": f"Bearer {get_igdb_token()}",
         "Client-ID": IGDB_CLIENT_ID,
@@ -65,7 +72,7 @@ def lookup_game_from_igdb(igdb_id: str) -> Dict:
     fields = "id,name,alternative_names.*,genres.*,release_dates.*,cover.*,screenshots.*,rating,rating_count,summary"
 
     game_dict = {}
-    body = f"fields {fields}; where id = {igdb_id};"
+    body = f"fields {fields}; where id = {igdb_id}; sort id asc;"
     response = requests.post(GAMES_URL, data=body, headers=headers)
     results = json.loads(response.content)
     if not results:

vrobbler/apps/videogames/migrations/0010_alter_videogame_genre.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.1.7 on 2023-04-18 02:33
+
+from django.db import migrations
+import taggit.managers
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0040_alter_scrobble_media_type"),
+        ("videogames", "0009_videogame_genre"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="videogame",
+            name="genre",
+            field=taggit.managers.TaggableManager(
+                blank=True,
+                help_text="A comma-separated list of tags.",
+                through="scrobbles.ObjectWithGenres",
+                to="scrobbles.Genre",
+                verbose_name="Tags",
+            ),
+        ),
+    ]

vrobbler/apps/videogames/migrations/0011_videogame_retroarch_name.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.7 on 2023-05-24 19:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("videogames", "0010_alter_videogame_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="videogame",
+            name="retroarch_name",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/videogames/models.py

@@ -6,6 +6,8 @@ from django.contrib.auth import get_user_model
 from django.db import models
 from django.urls import reverse
 from django_extensions.db.models import TimeStampedModel
+from imagekit.models import ImageSpecField
+from imagekit.processors import ResizeToFit
 from scrobbles.mixins import LongPlayScrobblableMixin
 from scrobbles.utils import get_scrobbles_for_media
 from videogames.igdb import lookup_game_id_from_gdb
@@ -34,6 +36,18 @@ class VideoGameCollection(TimeStampedModel):
     name = models.CharField(max_length=255)
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     cover = models.ImageField(upload_to="games/series-covers/", **BNULL)
+    cover_small = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    cover_medium = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
     igdb_id = models.IntegerField(**BNULL)
 
     def __str__(self):
@@ -72,9 +86,45 @@ class VideoGame(LongPlayScrobblableMixin):
     alternative_name = models.CharField(max_length=255, **BNULL)
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     cover = models.ImageField(upload_to="games/covers/", **BNULL)
+    cover_small = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    cover_medium = ImageSpecField(
+        source="cover",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
     screenshot = models.ImageField(upload_to="games/screenshots/", **BNULL)
+    screenshot_small = ImageSpecField(
+        source="screenshot",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    screenshot_medium = ImageSpecField(
+        source="screenshot",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
     summary = models.TextField(**BNULL)
     hltb_cover = models.ImageField(upload_to="games/hltb_covers/", **BNULL)
+    hltb_cover_small = ImageSpecField(
+        source="hltb_cover",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    hltb_cover_medium = ImageSpecField(
+        source="hltb_cover",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
     rating = models.FloatField(**BNULL)
     rating_count = models.IntegerField(**BNULL)
     release_date = models.DateTimeField(**BNULL)
@@ -84,6 +134,7 @@ class VideoGame(LongPlayScrobblableMixin):
     completionist_time = models.IntegerField(**BNULL)
     hltb_score = models.FloatField(**BNULL)
     platforms = models.ManyToManyField(VideoGamePlatform)
+    retroarch_name = models.CharField(max_length=255, **BNULL)
 
     def __str__(self):
         return self.title
@@ -96,9 +147,9 @@ class VideoGame(LongPlayScrobblableMixin):
     def primary_image_url(self) -> str:
         url = ""
         if self.cover:
-            url = self.cover.url
+            url = self.cover_medium.url
         if self.hltb_cover:
-            url = self.hltb_cover.url
+            url = self.hltb_cover_medium.url
         return url
 
     def get_absolute_url(self):
@@ -143,10 +194,11 @@ class VideoGame(LongPlayScrobblableMixin):
         if self.hltb_id and force_update:
             get_or_create_videogame(str(self.hltb_id), force_update)
 
-        if self.igdb_id:
+        if not self.igdb_id:
             # This almost never works without intervention
-            # self.igdb_id = lookup_game_id_from_gdb(self.title)
-            # self.save(update_fields=["igdb_id"])
+            self.igdb_id = lookup_game_id_from_gdb(self.title)
+
+        if self.igdb_id:
             load_game_data_from_igdb(self.id, self.igdb_id)
 
         if (not self.run_time_ticks or force_update) and self.main_story_time:

vrobbler/apps/videogames/retroarch.py

@@ -0,0 +1,178 @@
+import json
+import logging
+import os
+from datetime import datetime, timedelta
+from typing import List, Optional
+
+import pytz
+from dateutil.parser import ParserError, parse
+from django.apps import apps
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from scrobbles.utils import convert_to_seconds
+from videogames.models import VideoGame
+from videogames.scrapers import scrape_game_name_from_adb
+from videogames.utils import get_or_create_videogame
+from vrobbler.apps.scrobbles.exceptions import UserNotFound
+from vrobbler.apps.videogames.exceptions import GameNotFound
+
+logger = logging.getLogger(__name__)
+
+User = get_user_model()
+
+
+def load_game_data(directory_path: str, user_tz=None) -> dict:
+    """Given a path to a directory, cycle through each found lrtl file and
+    generate game data.
+
+    Example json file as follows:
+
+      Name: "Sonic The Hedgehog 2 (World).lrtl"
+
+      Contents:
+      {
+        "version": "1.0",
+        "runtime": "0:20:19",
+        "last_played": "2023-05-23 15:30:15"
+      }
+
+    """
+    directory = os.fsencode(directory_path)
+    games = {}
+    if not user_tz:
+        user_tz = settings.TIME_ZONE
+
+    for file in os.listdir(directory):
+        filename = os.fsdecode(file)
+        if not filename.endswith("lrtl"):
+            logger.info(f'Skipping "{filename}", not lrtl file')
+            continue
+
+        game_name = filename.split(".lrtl")[0].split(" (")[0]
+        with open("".join([directory_path, filename])) as f:
+            try:
+                games[game_name] = json.load(f)
+            except json.JSONDecodeError:
+                logger.warn(
+                    f"Could not decode JSOn for {game_name} and file {filename}"
+                )
+            # Convert runtime to seconds
+            games[game_name]["runtime"] = convert_to_seconds(
+                games[game_name]["runtime"]
+            )
+            # Convert last_played to datetime in user timezone
+            last_played_dt = parse(games.get(game_name).get("last_played"))
+            games[game_name]["last_played"] = user_tz.localize(last_played_dt)
+
+    return games
+
+
+def lookup_from_short_mame_name(game_name: str) -> Optional[VideoGame]:
+    logger.info(f"Received name {game_name}")
+    try:
+        mame_name = scrape_game_name_from_adb(game_name)
+    except GameNotFound as e:
+        logger.warning(e)
+        return
+
+    if mame_name:
+        logger.info(f"Found name {game_name}")
+        game_name = mame_name
+
+    return VideoGame.objects.filter(retroarch_name=game_name).first()
+
+
+def import_retroarch_lrtl_files(playlog_path: str, user_id: int) -> List[dict]:
+    """Given a path to Retroarch lrtl game log file data,
+    gather
+
+    For each found log file, we'll do:
+        1. Look up game, create if it doesn't exist
+        2. Check for existing scrobbles
+        3. Create new scrobble if last_played != last_scrobble.timestamp
+        4. Calculate scrobble time from runtime - last_scrobble.long_play_time
+    """
+    Scrobble = apps.get_model("scrobbles", "Scrobble")
+    user = User.objects.filter(pk=user_id).first()
+    if not user:
+        logger.warning(f"User ID {user_id} is not valid, cannot scrobble")
+        raise UserNotFound
+
+    game_logs = load_game_data(
+        playlog_path, pytz.timezone(user.profile.timezone)
+    )
+    found_game = None
+    new_scrobbles = []
+
+    for game_name, game_data in game_logs.items():
+        found_game = VideoGame.objects.filter(retroarch_name=game_name).first()
+
+        # If we didn't find by Retroarch name, check ArcadeDB
+        found_game_from_adb = None
+        if not found_game:
+            found_game_from_adb = lookup_from_short_mame_name(game_name)
+
+        # If we didn't find it on ADB, go to get_or_create
+        if not found_game and not found_game_from_adb:
+            try:
+                found_game = get_or_create_videogame(game_name)
+            except GameNotFound as e:
+                logger.warning(f"Game not found for: {e}")
+                continue
+
+            if found_game:
+                found_game.retroarch_name = game_name
+                found_game.save(update_fields=["retroarch_name"])
+
+        if not found_game:
+            logger.warning(f"No game found or created for {game_name}")
+            continue
+
+        # Found a game, check if scrobble exists
+        end_datetime = game_data.get("last_played")
+        found_scrobble = found_game.scrobble_set.filter(
+            stop_timestamp=end_datetime
+        )
+        if found_scrobble:
+            logger.info(f"Skipping scrobble for game {found_game.id}")
+            continue
+
+        last_scrobble = found_game.scrobble_set.last()
+
+        long_play_complete = None
+        if last_scrobble:
+            long_play_complete = last_scrobble.long_play_complete
+
+        # Default to 0 for delta, but if there's an past scrobble, use that
+        delta_runtime = 0
+        if last_scrobble:
+            delta_runtime = last_scrobble.long_play_seconds
+
+        playback_position_seconds = game_data["runtime"] - delta_runtime
+        timestamp = end_datetime - timedelta(seconds=playback_position_seconds)
+        if playback_position_seconds < 30:
+            logger.info(
+                f"Video game {found_game.id} played for less than 30 seconds, skipping"
+            )
+            continue
+
+        logger.info(f"Queued scrobble for game {found_game.id}")
+        new_scrobbles.append(
+            Scrobble(
+                video_game_id=found_game.id,
+                timestamp=timestamp,
+                stop_timestamp=end_datetime,
+                playback_position_seconds=playback_position_seconds,
+                played_to_completion=True,
+                in_progress=False,
+                long_play_seconds=game_data["runtime"],
+                long_play_complete=long_play_complete,
+                user_id=user_id,
+                source="Retroarch",
+                source_id="Imported from Retroarch play log file",
+                media_type=Scrobble.MediaType.VIDEO_GAME,
+            )
+        )
+    created_scrobbles = Scrobble.objects.bulk_create(new_scrobbles)
+    logger.info(f"Created {len(created_scrobbles)} scrobbles")
+    return new_scrobbles

vrobbler/apps/videogames/scrapers.py

@@ -0,0 +1,42 @@
+import logging
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+from vrobbler.apps.videogames.exceptions import GameNotFound
+
+logger = logging.getLogger(__name__)
+
+MAME_LOOKUP_URL = "http://adb.arcadeitalia.net/dettaglio_mame.php?game_name={query}&search_id=2"
+
+
+def _strip_and_clean(text):
+    return text.replace("\n", " ").rstrip().lstrip()
+
+
+def _get_title_from_soup(soup) -> Optional[int]:
+    title = None
+    try:
+        title = soup.find("h1", id="page_title").get_text()
+    except ValueError:
+        pass
+    return title
+
+
+def scrape_game_name_from_adb(name: str) -> str:
+    title = ""
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    url = MAME_LOOKUP_URL.format(query=name)
+    resp = requests.get(url, headers=headers)
+    if not resp.ok:
+        raise GameNotFound(f"Lookup failed with code {resp.status_code}")
+
+    soup = BeautifulSoup(resp.text, "html.parser")
+    title = _get_title_from_soup(soup).split(" (")[0].split(" / ")[0]
+
+    logger.info(f"Found title {title}")
+    if title == "Arcade Database":
+        title = ""
+
+    return title

vrobbler/apps/videogames/utils.py

@@ -3,37 +3,69 @@ from typing import Optional
 
 import requests
 from django.core.files.base import ContentFile
-from videogames.models import VideoGame, VideoGamePlatform
-
 from videogames.howlongtobeat import lookup_game_from_hltb
 from videogames.igdb import lookup_game_from_igdb
+from videogames.models import VideoGame, VideoGamePlatform
+
+from vrobbler.apps.videogames.exceptions import GameNotFound
 
 logger = logging.getLogger(__name__)
 
 
 def get_or_create_videogame(
-    name_or_id: str, force_update=False
+    name_or_id: str,
+    force_update: bool = False,
 ) -> Optional[VideoGame]:
     """Look up game by name or ID from HowLongToBeat"""
 
     game_dict = lookup_game_from_hltb(name_or_id)
 
+    if not game_dict:
+        game_dict = lookup_game_from_igdb(name_or_id)
+
     if not game_dict:
         return
 
+    # Create missing platforms and prep for loading after create
     platform_ids = []
-    for platform in game_dict.get("platforms"):
-        p, _created = VideoGamePlatform.objects.get_or_create(name=platform)
-        platform_ids.append(p.id)
-    game_dict.pop("platforms")
+    if "platforms" in game_dict.keys():
+        for platform in game_dict.get("platforms", []):
+            p, _created = VideoGamePlatform.objects.get_or_create(
+                name=platform
+            )
+            platform_ids.append(p.id)
+        game_dict.pop("platforms")
 
-    game, game_created = VideoGame.objects.get_or_create(
-        hltb_id=game_dict.get("hltb_id")
-    )
+    cover_url = game_dict.pop("cover_url")
+
+    screenshot_url = ""
+    if "screenshot_url" in game_dict.keys():
+        screenshot_url = game_dict.pop("screenshot_url")
+
+    genres = []
+    if "genres" in game_dict.keys():
+        genres = game_dict.pop("genres")
+
+    title = game_dict.get("title")
+    if not title:
+        raise GameNotFound(name_or_id)
+
+    hltb_id = game_dict.get("hltb_id")
+    igdb_id = game_dict.get("igdb_id")
+    if hltb_id:
+        game, game_created = VideoGame.objects.get_or_create(
+            hltb_id=hltb_id,
+            title=title,
+        )
+    elif igdb_id:
+        game, game_created = VideoGame.objects.get_or_create(
+            igdb_id=igdb_id,
+            title=title,
+        )
+    else:
+        game, game_created = VideoGame.objects.get_or_create(title=title)
 
     if game_created or force_update:
-        cover_url = game_dict.pop("cover_url")
-
         VideoGame.objects.filter(pk=game.id).update(**game_dict)
         game.refresh_from_db()
 
@@ -41,6 +73,15 @@ def get_or_create_videogame(
         if platform_ids:
             game.platforms.add(*platform_ids)
 
+        if genres:
+            game.genre.add(*genres)
+
+        if not game.screenshot and screenshot_url:
+            r = requests.get(screenshot_url)
+            if r.status_code == 200:
+                fname = f"{game.title}_{game.uuid}.jpg"
+                game.screenshot.save(fname, ContentFile(r.content), save=True)
+
         # Go get cover image if the URL is present
         if cover_url and not game.hltb_cover:
             headers = {"User-Agent": "Vrobbler 0.11.12"}
@@ -86,13 +127,13 @@ def load_game_data_from_igdb(
 
     game.genre.add(*genres)
 
-    if not game.screenshot:
+    if not game.screenshot and screenshot_url:
         r = requests.get(screenshot_url)
         if r.status_code == 200:
             fname = f"{game.title}_{game.uuid}.jpg"
             game.screenshot.save(fname, ContentFile(r.content), save=True)
 
-    if not game.cover:
+    if not game.cover and cover_url:
         r = requests.get(cover_url)
         if r.status_code == 200:
             fname = f"{game.title}_{game.uuid}.jpg"