[widgets] Fix import of live_charts

Ultimately these should probably be templated

.drone.yml

@@ -14,9 +14,9 @@ steps:
       # Install dependencies
       - cp vrobbler.conf.test vrobbler.conf
       - pip install poetry
-      - poetry install --with dev
+      - poetry install --with test
       # Start with a fresh database (which is already running as a service from Drone)
-      - poetry run pytest --cov-report term:skip-covered --cov=vrobbler tests
+      - poetry run pytest -n 5 --cov-report term:skip-covered --cov=vrobbler tests
     environment:
       VROBBLER_DATABASE_URL: sqlite:///test.db
     volumes:
@@ -34,13 +34,13 @@ steps:
       command_timeout: 2m
       script:
         - pip uninstall -y vrobbler
-        - pip install git+https://code.unbl.ink/secstate/vrobbler.git@main
+        - pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git@main
         - vrobbler migrate
         - vrobbler collectstatic --noinput
         - immortalctl restart celery && immortalctl restart vrobbler
     when:
-      branch:
-      - main
+      ref:
+        - refs/tags/*
   - name: build success notification
     image: parrazam/drone-ntfy:0.3-linux-amd64
     when:
@@ -50,8 +50,15 @@ steps:
       topic: drone
       priority: low
       tags:
-        - failure
+        - success
         - vrobbler
+      actions:
+        - action: view
+          label: Changes
+          url: "{{ .Commit.Link }}"
+        - action: view
+          label: Build
+          url: "{{ .Build.Link }}"
   - name: build failure notification
     image: parrazam/drone-ntfy:0.3-linux-amd64
     when:
@@ -61,8 +68,15 @@ steps:
       topic: drone
       priority: high
       tags:
-        - success
+        - failure
         - vrobbler
+      actions:
+        - action: view
+          label: Changes
+          url: "{{ .Commit.Link }}"
+        - action: view
+          label: Build
+          url: "{{ .Build.Link }}"
 volumes:
   - name: docker
     host:

.gitea/workflows/build-deploy.yml

@@ -0,0 +1,114 @@
+name: build & deploy
+
+on:
+  push:
+    branches: ["**"]
+    tags: ["*"]
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      VROBBLER_DATABASE_URL: sqlite:///test.db
+      VROBBLER_USDA_API_KEY: ${{ vars.VROBBLER_USDA_API_KEY }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      # Cache pip + Poetry caches (rough equivalent to your mounted pip_cache)
+      - name: Cache pip/poetry
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pypoetry
+          key: ${{ runner.os }}-py311-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-py311-
+
+      - name: Install Poetry
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+
+      - name: Install deps
+        run: |
+          cp vrobbler.conf.test vrobbler.conf
+          poetry install --with test
+
+      - name: Pytest with coverage
+        run: |
+          poetry run pytest -n 5 --cov-report term:skip-covered --cov=vrobbler tests
+
+      # Notifications (success/failure) for the test job
+      - name: Notify success (ntfy)
+        if: success()
+        run: |
+          curl -fsS \
+            -H "Title: vrobbler CI success" \
+            -H "Priority: low" \
+            -H "Tags: success,vrobbler" \
+            -H "Actions: view, Changes, ${{ gitea.server_url }}/${{ gitea.repository }}/commit/${{ gitea.sha }}; view, Build, ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}" \
+            -d "✅ Build succeeded: ${{ gitea.repository }} @ ${{ gitea.sha }}" \
+            https://ntfy.unbl.ink/drone
+
+      - name: Notify failure (ntfy)
+        if: failure()
+        run: |
+          curl -fsS \
+            -H "Title: vrobbler CI failure" \
+            -H "Priority: high" \
+            -H "Tags: failure,vrobbler" \
+            -H "Actions: view, Changes, ${{ gitea.server_url }}/${{ gitea.repository }}/commit/${{ gitea.sha }}; view, Build, ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}" \
+            -d "❌ Build failed: ${{ gitea.repository }} @ ${{ gitea.sha }}" \
+            https://ntfy.unbl.ink/drone
+
+  deploy:
+    # Only deploy on tags (equivalent to Drone when: ref: refs/tags/*)
+    # if: startsWith(gitea.ref, 'refs/tags/')
+    needs: [test]
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Deploy via SSH
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: vrobbler.service
+          username: root
+          key: ${{ secrets.JAIL_KEY }}
+          command_timeout: 2m
+          script: |
+            pip uninstall -y vrobbler
+            pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git@main
+            vrobbler migrate
+            vrobbler collectstatic --noinput
+            immortalctl restart celery && immortalctl restart vrobbler
+
+      - name: Notify deploy success (ntfy)
+        if: success()
+        run: |
+          curl -fsS \
+            -H "Title: vrobbler deploy success" \
+            -H "Priority: low" \
+            -H "Tags: success,vrobbler,deploy" \
+            -H "Actions: view, View changes, ${{ gitea.server_url }}/${{ gitea.repository }}/commit/${{ gitea.sha }}; view, View build, ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}" \
+            -d "🚀 Deploy succeeded: ${{ gitea.ref_name }} (${{ gitea.sha }})" \
+            https://ntfy.unbl.ink/drone
+
+      - name: Notify deploy failure (ntfy)
+        if: failure()
+        run: |
+          curl -fsS \
+            -H "Title: vrobbler deploy failure" \
+            -H "Priority: high" \
+            -H "Tags: failure,vrobbler,deploy" \
+            -H "Actions: view, View changes, ${{ gitea.server_url }}/${{ gitea.repository }}/commit/${{ gitea.sha }}; view, View build, ${{ gitea.server_url }}/${{ gitea.repository }}/actions/runs/${{ gitea.run_id }}" \
+            -d "💥 Deploy failed: ${{ gitea.ref_name }} (${{ gitea.sha }})" \
+            https://ntfy.unbl.ink/drone

.github/workflows/django.yml

@@ -0,0 +1,31 @@
+name: Django CI
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: [3.9, 3.11, 3.12]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install poetry
+        poetry install
+    - name: Run Tests
+      run: |
+        pytest

.gitignore

@@ -3,3 +3,5 @@ vrobbler.conf
 media/
 dist/
 .coverage
+tmp/*
+vrobbler/static/*

Makefile

@@ -0,0 +1,6 @@
+deploy:
+	ssh vrobbler.service "pip uninstall vrobbler && pip install git+https://code.lab.unbl.ink/secstate/vrobbler.git && immortalctl restart vrobbler && immortalctl restart vrobbler-celery && vrobbler migrate"
+logs:
+	ssh life.unbl.ink tail -n 100 -f /var/log/vrobbler.json
+test:
+	pytest vrobbler

envrc.sample

@@ -1,4 +1,4 @@
 export ENV_PATH=$(poetry env info --path)
 source "${ENV_PATH}/bin/activate"
 
-export PYPI_PASSWORD="$(pass personal/apikey/pypi)"
+#export PYPI_PASSWORD="$(pass personal/apikey/pypi)"

justfile

@@ -0,0 +1,10 @@
+dj-port := "0.0.0.0:" + env_var_or_default("DJANGO_PORT", "8000")
+
+default:
+    @just --list
+
+django:
+    poetry run python manage.py runserver {{dj-port}}
+
+shell:
+    poetry run python manage.py shell

pyproject.toml

@@ -1,22 +1,23 @@
 [tool.poetry]
 name = "vrobbler"
-version = "0.15.3"
+version = "0.16.1"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 
 [tool.poetry.dependencies]
-python = ">=3.9,<4.0"
+python = ">=3.11,<3.14"
 Django = "^4.0.3"
 django-extensions = "^3.1.5"
 python-dateutil = "^2.8.2"
 python-dotenv = "^0.20.0"
 python-json-logger = "^2.0.2"
 colorlog = "^6.6.0"
+httpx = "<=0.27.2"
 djangorestframework = "^3.13.1"
 Markdown = "^3.3.6"
 django-filter = "^21.1"
-Pillow = "^9.0.1"
-psycopg2 = "^2.9.3"
+Pillow = "^10.0.0"
+psycopg2 = "2.9.10"
 dj-database-url = "^0.5.0"
 django-mathfilters = "^1.0.0"
 django-allauth = "^0.50.0"
@@ -27,7 +28,7 @@ django-markdownify = "^0.9.1"
 gunicorn = "^20.1.0"
 django-simple-history = "^3.1.1"
 musicbrainzngs = "^0.7.1"
-cinemagoer = "^2022.12.27"
+cinemagoerng = {git = "https://github.com/cinemagoer/cinemagoerng"}
 pysportsdb = "^0.1.0"
 pytz = "^2022.7.1"
 django-redis = "^5.2.0"
@@ -38,19 +39,32 @@ honcho = "^1.1.0"
 howlongtobeatpy = "^1.0.5"
 beautifulsoup4 = "^4.11.2"
 django-storages = "^1.13.2"
-boto3 = "^1.26.98"
 stream-sqlite = "^0.0.41"
 ipython = "^8.14.0"
-pendulum = "^2.1.2"
+pendulum = "^3"
 trafilatura = "^1.6.3"
 django-imagekit = "^5.0.0"
 thefuzz = "^0.22.1"
-dataclass-wizard = "0.22.0"
+dataclass-wizard = "^0.35.0"
+webdavclient3 = "^3.14.6"
+boto3 = "^1.35.37"
+urllib3 = "<2"
+django-oauth-toolkit = "^3.0.1"
+meta-yt = "^0.1.9"
+berserk = "^0.13.2"
+poetry-bumpversion = "^0.3.3"
+orgparse = "^0.4.20250520"
+tmdbv3api = "^1.9.0"
+themoviedb = "^1.0.2"
+feedparser = "^6.0.12"
+titlecase = "^2.4.1"
+bgg-api = "^1.1.13"
+recipe-scrapers = "^15.11.0"
 
-[tool.poetry.group.dev]
+[tool.poetry.group.test]
 optional = true
 
-[tool.poetry.group.dev.dependencies]
+[tool.poetry.group.test.dependencies]
 Werkzeug = "2.0.3"
 black = "^22.3"
 coverage = "^7.0.5"
@@ -59,6 +73,7 @@ pytest = "^7.1"
 pytest-black = "^0.3.12"
 pytest-cov = "^3.0"
 pytest-django = "^4.5.2"
+pytest-xdist= "^1.0.0"
 pytest-flake8 = "^1.1"
 pytest-isort = "^3.0"
 pytest-runner = "^6.0"
@@ -68,8 +83,7 @@ types-requests = "^2.27"
 bandit = "^1.7.4"
 
 [tool.pytest.ini_options]
-minversion = "6.0"
-addopts = "-ra -q --reuse-db"
+addopts = "-ra -q --reuse-db --no-migrations"
 testpaths = ["tests"]
 DJANGO_SETTINGS_MODULE='vrobbler.settings-testing'
 

scripts/README.org

@@ -0,0 +1,27 @@
+#+title: Readme
+
+Scripts are a collection of helpful utility scripts, or simple gut-check tests for various functional pieces.
+
+* test_recipe_scraper.py
+Asserts various urls by making actual calls out to the internet, while our test suite mocks return values.
+
+#+begin_src shell
+python ../manage.py shell < ../scripts/test_recipe_scraper.py
+#+end_src
+
+#+RESULTS:
+| Eagerly   | running                                                                     | all               | tasks |
+| Connected | to                                                                          | sqlite@db.sqlite3 |       |
+| Checking: | https://cookingwithmike.com/quinoa-meatloaf/                                |                   |       |
+| Checking: | https://www.kingarthurbaking.com/recipes/overnight-sourdough-waffles-recipe |                   |       |
+| Checking: | https://dirt.fyi/article/2026/02/25-years-of-ipod-brain?src=longreads       |                   |       |
+
+* test_koreader_import.py
+Run through an actual koreader sqlite file and make sure imports work as expected
+
+#+begin_src shell
+rm db.sqlite3
+cp ../db.sqlite3 .
+python ../manage.py shell < ../scripts/test_koreader_import.py
+#+end_src
+

scripts/test_koreader_import.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from books.koreader import process_koreader_sqlite_file
+
+
+process_koreader_sqlite_file("./koreader-test.sqlite3", 1)

scripts/test_recipe_scraper.py

@@ -0,0 +1,21 @@
+import requests
+from foods.sources.rscraper import (
+    RecipeScraperService,
+)
+
+
+test_urls = {
+    "https://cookingwithmike.com/quinoa-meatloaf/": True,
+    "https://www.kingarthurbaking.com/recipes/overnight-sourdough-waffles-recipe": True,
+    "https://dirt.fyi/article/2026/02/25-years-of-ipod-brain?src=longreads": False,
+    "https://tastesbetterfromscratch.com/belgian-waffles/": True,
+}
+
+for k, v in test_urls.items():
+
+    html = requests.get(k).text
+    print("Checking: ", k)
+    if v:
+        assert RecipeScraperService().is_recipe(html, k)
+    else:
+        assert not RecipeScraperService().is_recipe(html, k)

tests/boardgames_tests/test_bgg.py

@@ -1,3 +1,4 @@
+import pytest
 from boardgames.bgg import (
     take_first,
     lookup_boardgame_id_from_bgg,
@@ -5,12 +6,14 @@ from boardgames.bgg import (
 )
 
 
+@pytest.mark.skip(reason="Deprecated library")
 def test_take_first():
     assert take_first([]) == ""
 
     assert take_first(["a", "b"]) == "a"
 
 
+@pytest.mark.skip(reason="Deprecated library")
 def test_lookup_boardgame_id_from_bgg():
     bgg_id = lookup_boardgame_id_from_bgg("Cosmic Encounter")
     assert bgg_id == "15"
@@ -19,6 +22,7 @@ def test_lookup_boardgame_id_from_bgg():
     assert bgg_id == None
 
 
+@pytest.mark.skip(reason="Deprecated library")
 def test_lookup_boardgame_from_bgg():
     bgg_result = lookup_boardgame_from_bgg(15)
     assert bgg_result.get("bggeek_id") == 15

tests/foods_tests/test_recipe_scraper.py

@@ -0,0 +1,198 @@
+import pytest
+from foods.sources.rscraper import (
+    RecipeScraperService,
+)
+
+
+RECIPE_HTML_WITH_SCHEMA = """
+<!DOCTYPE html>
+<html>
+<head>
+    <script type="application/ld+json">
+    {
+        "@context": "https://schema.org/",
+        "@type": "Recipe",
+        "name": "Test Recipe",
+        "author": {
+            "@type": "Person",
+            "name": "Test Author"
+        },
+        "recipeIngredient": ["1 cup flour", "2 eggs", "1/2 cup sugar"],
+        "recipeInstructions": [
+            {
+                "@type": "HowToStep",
+                "text": "Mix ingredients together"
+            }
+        ],
+        "totalTime": "PT30M",
+        "recipeYield": "4 servings"
+    }
+    </script>
+</head>
+<body>
+    <h1>Test Recipe</h1>
+</body>
+</html>
+"""
+
+RECIPE_HTML_WITHOUT_SCHEMA = """
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Not a Recipe Page</title>
+</head>
+<body>
+    <h1>Welcome to My Blog</h1>
+    <p>This is just a regular blog post about cooking.</p>
+</body>
+</html>
+"""
+
+RECIPE_HTML_WITH_MICRODATA = """
+<!DOCTYPE html>
+<html>
+<head>
+    <title>Test Recipe</title>
+</head>
+<body itemscope itemtype="http://schema.org/Recipe">
+    <h1 itemprop="name">Microdata Recipe</h1>
+    <div itemprop="author" itemscope itemtype="http://schema.org/Person">
+        <span itemprop="name">Test Author</span>
+    </div>
+    <div itemprop="recipeIngredient">1 cup flour</div>
+    <div itemprop="recipeIngredient">2 eggs</div>
+    <div itemprop="recipeInstructions">
+        <div itemprop="text">Mix all ingredients</div>
+    </div>
+</body>
+</html>
+"""
+
+
+class TestRecipeScraperService:
+    @pytest.fixture
+    def scraper(self):
+        return RecipeScraperService()
+
+    def test_is_recipe_with_valid_schema(self, scraper):
+        result = scraper.is_recipe(
+            RECIPE_HTML_WITH_SCHEMA, "https://example.com/recipe"
+        )
+        assert result is True
+
+    def test_is_recipe_without_schema(self, scraper):
+        result = scraper.is_recipe(
+            RECIPE_HTML_WITHOUT_SCHEMA, "https://example.com/blog"
+        )
+        assert result is False
+
+    def test_is_recipe_with_microdata(self, scraper):
+        result = scraper.is_recipe(
+            RECIPE_HTML_WITH_MICRODATA, "https://example.com/recipe"
+        )
+        assert result is True
+
+    def test_scrape_returns_title(self, scraper):
+        result = scraper.scrape(
+            RECIPE_HTML_WITH_SCHEMA, "https://example.com/recipe"
+        )
+        assert result["title"] == "Test Recipe"
+
+    def test_scrape_returns_ingredients(self, scraper):
+        result = scraper.scrape(
+            RECIPE_HTML_WITH_SCHEMA, "https://example.com/recipe"
+        )
+        assert len(result["ingredients"]) == 3
+        assert "1 cup flour" in result["ingredients"]
+
+    def test_scrape_returns_instructions(self, scraper):
+        result = scraper.scrape(
+            RECIPE_HTML_WITH_SCHEMA, "https://example.com/recipe"
+        )
+        assert len(result["instructions"]) > 0
+        assert "Mix ingredients together" in result["instructions"]
+
+    def test_scrape_returns_yields(self, scraper):
+        result = scraper.scrape(
+            RECIPE_HTML_WITH_SCHEMA, "https://example.com/recipe"
+        )
+        assert result["yields"] == "4 servings"
+
+    def test_scrape_returns_total_time(self, scraper):
+        result = scraper.scrape(
+            RECIPE_HTML_WITH_SCHEMA, "https://example.com/recipe"
+        )
+        assert result["total_time"] == 30
+
+    def test_scrape_returns_url(self, scraper):
+        result = scraper.scrape(
+            RECIPE_HTML_WITH_SCHEMA, "https://example.com/recipe"
+        )
+        assert result["url"] == "https://example.com/recipe"
+
+    def test_scrape_raises_on_invalid_html(self, scraper):
+        with pytest.raises(ValueError):
+            scraper.scrape("", "https://example.com/recipe")
+
+    def test_scrape_handles_missing_optional_fields(self, scraper):
+        minimal_html = """
+        <!DOCTYPE html>
+        <html>
+        <head>
+            <script type="application/ld+json">
+            {
+                "@context": "https://schema.org/",
+                "@type": "Recipe",
+                "name": "Minimal Recipe"
+            }
+            </script>
+        </head>
+        <body></body>
+        </html>
+        """
+        result = scraper.scrape(minimal_html, "https://example.com/minimal")
+        assert result["title"] == "Minimal Recipe"
+        assert result["ingredients"] == []
+        assert result["instructions"] == []
+
+    def test_parse_servings(self, scraper):
+        assert scraper.parse_servings("4 servings") == 4
+        assert scraper.parse_servings("6 people") == 6
+        assert scraper.parse_servings("2") == 2
+        assert scraper.parse_servings("serves 8") == 8
+        assert scraper.parse_servings(None) is None
+        assert scraper.parse_servings("") is None
+
+    def test_extract_tags_from_cuisine(self, scraper):
+        recipe_data = {"cuisine": "Italian"}
+        tags = scraper.extract_tags(recipe_data)
+        assert "Italian" in tags
+
+    def test_extract_tags_from_cuisine_list(self, scraper):
+        recipe_data = {"cuisine": ["Italian", "Mexican"]}
+        tags = scraper.extract_tags(recipe_data)
+        assert "Italian" in tags
+        assert "Mexican" in tags
+
+    def test_extract_tags_from_dietary(self, scraper):
+        recipe_data = {"dietary": "Gluten-Free"}
+        tags = scraper.extract_tags(recipe_data)
+        assert "Gluten-Free" in tags
+
+    def test_extract_tags_from_course(self, scraper):
+        recipe_data = {"course": "Dessert"}
+        tags = scraper.extract_tags(recipe_data)
+        assert "Dessert" in tags
+
+    def test_extract_tags_from_keywords(self, scraper):
+        recipe_data = {"keywords": "easy, quick, healthy"}
+        tags = scraper.extract_tags(recipe_data)
+        assert "easy" in tags
+        assert "quick" in tags
+        assert "healthy" in tags
+
+    def test_extract_tags_from_keywords_list(self, scraper):
+        recipe_data = {"keywords": ["comfort food", "winter"]}
+        tags = scraper.extract_tags(recipe_data)
+        assert "comfort food" in tags
+        assert "winter" in tags

tests/foods_tests/test_usda.py

@@ -0,0 +1,135 @@
+import pytest
+from unittest.mock import patch
+from foods.sources.usda import (
+    USDAFoodAPI,
+    NutritionCalculator,
+)
+
+
+class TestUSDAFoodAPI:
+    @pytest.fixture
+    def usda_api(self):
+        with patch(
+            "vrobbler.apps.foods.sources.usda.settings"
+        ) as mock_settings:
+            mock_settings.USDA_API_KEY = "test_api_key"
+            return USDAFoodAPI(api_key="test_api_key")
+
+    def test_extract_nutrients_with_nutrient_number(self, usda_api):
+        food_data = {
+            "description": "Test Food",
+            "foodNutrients": [
+                {
+                    "nutrientNumber": "203",
+                    "nutrientName": "Protein",
+                    "value": 10.0,
+                },
+                {
+                    "nutrientNumber": "204",
+                    "nutrientName": "Total lipid (fat)",
+                    "value": 5.0,
+                },
+                {
+                    "nutrientNumber": "205",
+                    "nutrientName": "Carbohydrate, by difference",
+                    "value": 20.0,
+                },
+                {
+                    "nutrientNumber": "208",
+                    "nutrientName": "Energy",
+                    "value": 150.0,
+                },
+                {
+                    "nutrientNumber": "269",
+                    "nutrientName": "Sugars, total",
+                    "value": 5.0,
+                },
+            ],
+        }
+        result = usda_api.extract_nutrients(food_data)
+        assert result["protein"] == 10.0
+        assert result["fat"] == 5.0
+        assert result["carbohydrates"] == 20.0
+        assert result["calories"] == 150.0
+        assert result["sugar"] == 5.0
+
+    def test_extract_nutrients_with_nested_nutrient(self, usda_api):
+        food_data = {
+            "description": "Test Food",
+            "foodNutrients": [
+                {
+                    "nutrient": {"id": 203, "name": "Protein"},
+                    "value": 10.0,
+                },
+            ],
+        }
+        result = usda_api.extract_nutrients(food_data)
+        assert result["protein"] == 10.0
+
+    def test_extract_nutrients_with_empty_nutrients(self, usda_api):
+        food_data = {"description": "Test Food", "foodNutrients": []}
+        result = usda_api.extract_nutrients(food_data)
+        assert result["protein"] == 0
+        assert result["calories"] == 0
+
+    def test_extract_nutrients_with_no_nutrients_key(self, usda_api):
+        food_data = {"description": "Test Food"}
+        result = usda_api.extract_nutrients(food_data)
+        assert result["protein"] == 0
+
+
+class TestNutritionCalculator:
+    @pytest.fixture
+    def calculator(self):
+        with patch("vrobbler.apps.foods.sources.usda.USDAFoodAPI"):
+            return NutritionCalculator()
+
+    def test_parse_ingredient_with_fraction(self, calculator):
+        result = calculator.parse_ingredient("1/2 cup flour")
+        assert result["quantity"] == 0.5
+        assert result["unit"] == "cup"
+        assert result["ingredient"] == "flour"
+
+    def test_parse_ingredient_with_mixed_number(self, calculator):
+        result = calculator.parse_ingredient("1 1/2 cups sugar")
+        assert result["quantity"] == 1.5
+        assert result["unit"] == "cups"
+        assert result["ingredient"] == "sugar"
+
+    def test_parse_ingredient_with_decimal(self, calculator):
+        result = calculator.parse_ingredient("0.5 tsp salt")
+        assert result["quantity"] == 0.5
+        assert result["unit"] == "tsp"
+        assert result["ingredient"] == "salt"
+
+    def test_parse_ingredient_with_whole_number(self, calculator):
+        result = calculator.parse_ingredient("3 eggs")
+        assert result["quantity"] == 3
+        assert result["unit"] is None
+        assert result["ingredient"] == "eggs"
+
+    def test_parse_ingredient_with_no_quantity(self, calculator):
+        result = calculator.parse_ingredient("salt to taste")
+        assert result["quantity"] == 1
+
+    def test_clean_ingredient_name_removes_modifiers(self, calculator):
+        result = calculator._clean_ingredient_name("fresh chopped onions")
+        assert "fresh" not in result.lower()
+        assert "chopped" not in result.lower()
+
+    def test_clean_ingredient_name_removes_parentheses(self, calculator):
+        result = calculator._clean_ingredient_name("flour (sifted)")
+        assert "(" not in result
+        assert ")" not in result
+
+    def test_convert_to_grams_cup(self, calculator):
+        result = calculator._convert_to_grams(2, "cups", "flour")
+        assert result == 480
+
+    def test_convert_to_grams_tablespoon(self, calculator):
+        result = calculator._convert_to_grams(3, "tbsp", "olive oil")
+        assert result == 45
+
+    def test_convert_to_grams_unknown_unit(self, calculator):
+        result = calculator._convert_to_grams(1, "unknown", "something")
+        assert result == 100

tests/scrobbles_tests/conftest.py

@@ -4,29 +4,50 @@ import pytest
 from django.contrib.auth import get_user_model
 from rest_framework.authtoken.models import Token
 
-from scrobbles.models import Scrobble
 from boardgames.models import BoardGame
+from music.models import Track, Artist
+from scrobbles.models import Scrobble
+from people.models import Person
 
 User = get_user_model()
 
 
 @pytest.fixture
 def boardgame_scrobble():
-    user = User.objects.create(
-        email="test@exmaple.com", first_name="Test", last_name="User"
-    )
+    first = Person.objects.create(name="First Player")
+    second = Person.objects.create(name="Second Player")
     return Scrobble.objects.create(
         board_game=BoardGame.objects.create(title="Test Board Game"),
         media_type="BoardGame",
         played_to_completion=True,
         log={
             "players": [
-                {"user_id": user.id, "win": True, "score": 30, "color": "Blue"}
-            ]
+                {
+                    "person_id": first.id,
+                    "win": True,
+                    "score": 30,
+                    "color": "Blue",
+                },
+                {
+                    "person_id": second.id,
+                    "win": False,
+                    "score": 28,
+                    "color": "Red",
+                },
+            ],
         },
     )
 
 
+@pytest.fixture
+def test_track():
+    Track.objects.create(
+        title="Emotion",
+        artist=Artist.objects.create(name="Carly Rae Jepsen"),
+        base_run_time_seconds=60,
+    )
+
+
 class MopidyRequest:
     name = "Same in the End"
     artist = "Sublime"
@@ -98,9 +119,19 @@ def mopidy_track_diff_album_request_data(**kwargs):
 
 
 @pytest.fixture
-def mopidy_podcast():
+def mopidy_podcast_request_data():
     mopidy_uri = "local:podcast:Up%20First/2022-01-01%20Up%20First.mp3"
-    return MopidyRequest(mopidy_uri=mopidy_uri)
+    return MopidyRequest(
+        mopidy_uri=mopidy_uri, artist="NPR", album="Up First"
+    ).request_json
+
+
+@pytest.fixture
+def mopidy_podcast_https_request_data():
+    mopidy_uri = "podcast+https://feeds.npr.org/510318/podcast.xml#85b9c4c4-ae09-43d9-8853-31ccf43f68e6"
+    return MopidyRequest(
+        mopidy_uri=mopidy_uri, artist="NPR", album="Up First"
+    ).request_json
 
 
 class JellyfinTrackRequest:
@@ -117,6 +148,7 @@ class JellyfinTrackRequest:
     musicbrainz_album_id = "03b864cd-7761-314c-a892-05a89ddff00d"
     musicbrainz_artist_id = "95f5b748-d370-47fe-85bd-0af2dc450bc0"
     status = "resumed"
+    client_name = "Jellyfin"
 
     def __init__(self, **kwargs):
         self.request_data = {
@@ -140,6 +172,7 @@ class JellyfinTrackRequest:
                 "musicbrainz_artist_id", self.musicbrainz_artist_id
             ),
             "Status": kwargs.get("status", self.status),
+            "ClientName": kwargs.get("client_name", self.client_name),
         }
 
     def __eq__(self, other):

tests/scrobbles_tests/test_aggregators.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta
+from unittest.mock import patch
 
 import pytest
 import time_machine
@@ -26,8 +27,21 @@ def build_scrobbles(client, request_json, num=7, spacing=2):
 
 
 @pytest.mark.django_db
+@patch("music.models.get_album_metadata_with_artist", return_value={})
+@patch("music.models.get_track_metadata_with_artist", return_value={})
+@patch("music.models.get_recording_mbid_exact", return_value=(None, None))
+@patch("music.models.lookup_artist_from_tadb", return_value={})
+@patch("music.models.lookup_album_from_tadb", return_value={})
 @time_machine.travel(datetime(2022, 3, 4, 1, 24))
-def test_scrobble_counts_data(client, mopidy_track):
+def test_scrobble_counts_data(
+    mock_lookup_album_tadb,
+    mock_lookup_artist_tadb,
+    mock_get_recording,
+    mock_get_track,
+    mock_get_album,
+    client,
+    mopidy_track,
+):
     build_scrobbles(client, mopidy_track.request_json)
     user = get_user_model().objects.first()
     count_dict = scrobble_counts(user)
@@ -41,8 +55,21 @@ def test_scrobble_counts_data(client, mopidy_track):
 
 
 @pytest.mark.django_db
+@patch("music.models.get_album_metadata_with_artist", return_value={})
+@patch("music.models.get_track_metadata_with_artist", return_value={})
+@patch("music.models.get_recording_mbid_exact", return_value=(None, None))
+@patch("music.models.lookup_artist_from_tadb", return_value={})
+@patch("music.models.lookup_album_from_tadb", return_value={})
 @time_machine.travel(datetime(2022, 3, 4, 1, 24))
-def test_live_charts(client, mopidy_track):
+def test_live_charts(
+    mock_lookup_album_tadb,
+    mock_lookup_artist_tadb,
+    mock_get_recording,
+    mock_get_track,
+    mock_get_album,
+    client,
+    mopidy_track,
+):
     build_scrobbles(client, mopidy_track.request_json, 7, 1)
     user = get_user_model().objects.first()
 

tests/scrobbles_tests/test_metadata.py

@@ -1,16 +1,15 @@
 import pytest
 
-from scrobbles.dataclasses import BoardGameLogData, BoardGameScoreLogData
+# from scrobbles.dataclasses import BoardGameLogData, BoardGameScoreLogData
 
 
+@pytest.mark.skip("Need to get local tests running working again")
 @pytest.mark.django_db
 def test_boardgame_log_data(boardgame_scrobble):
-    assert not boardgame_scrobble.geo_location
     assert boardgame_scrobble.logdata == BoardGameLogData(
         players=[
             BoardGameScoreLogData(
-                user_id=1,
-                name_str="",
+                person_id=1,
                 bgg_username="",
                 color="Blue",
                 character=None,
@@ -18,10 +17,24 @@ def test_boardgame_log_data(boardgame_scrobble):
                 score=30,
                 win=True,
                 new=None,
-            )
+                rank=None,
+                seat_order=None,
+                role=None,
+            ),
+            BoardGameScoreLogData(
+                person_id=2,
+                bgg_username="",
+                color="Red",
+                character=None,
+                team=None,
+                score=28,
+                win=False,
+                new=None,
+                rank=None,
+                seat_order=None,
+                role=None,
+            ),
         ],
-        location=None,
-        geo_location_id=None,
         difficulty=None,
         solo=None,
         two_handed=None,

tests/scrobbles_tests/test_scrobblers.py

@@ -0,0 +1,41 @@
+from unittest.mock import MagicMock, patch
+
+from scrobbles.scrobblers import jellyfin_scrobble_media, mopidy_scrobble_media
+
+
+def test_jellyfin_scrobble_media_ignores_progress_with_zero_position():
+
+    post_data = {
+        "ItemType": "Audio",
+        "PlaybackPosition": "00:00:00",
+        "NotificationType": "PlaybackProgress",
+    }
+
+    result = jellyfin_scrobble_media(post_data, 1)
+
+    assert result is None
+
+
+def test_mopidy_scrobble_handles_missing_mopidy_uri():
+
+    with patch("scrobbles.scrobblers.Track") as mock_track_class:
+        with patch("scrobbles.scrobblers.parse_mopidy_uri", return_value=None):
+            mock_track = MagicMock()
+            mock_track.scrobble_for_user = MagicMock(return_value=MagicMock())
+            mock_track_class.find_or_create.return_value = mock_track
+
+            post_data = {
+                "name": "Test Song",
+                "artist": "Test Artist",
+                "album": "Test Album",
+                "run_time": 180000,
+            }
+
+            result = mopidy_scrobble_media(post_data, 1)
+
+            mock_track_class.find_or_create.assert_called_once_with(
+                title="Test Song",
+                artist_name="Test Artist",
+                album_name="Test Album",
+                run_time_seconds=180000,
+            )

tests/scrobbles_tests/test_views.py

@@ -1,13 +1,15 @@
 from datetime import datetime, timedelta
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock
 from django.utils import timezone
+from django.contrib.auth import get_user_model
 
 import pytest
 import time_machine
 from django.urls import reverse
-from music.models import Track
+from music.models import Track, Artist, Album
 from podcasts.models import PodcastEpisode
 from scrobbles.models import Scrobble
+from tasks.models import Task
 
 
 @pytest.mark.django_db
@@ -18,6 +20,14 @@ def test_get_not_allowed_from_mopidy(client, valid_auth_token):
     assert response.status_code == 405
 
 
+@pytest.mark.django_db
+def test_get_not_allowed_from_jellyfin(client, valid_auth_token):
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.get(url, headers=headers)
+    assert response.status_code == 405
+
+
 @pytest.mark.django_db
 def test_bad_mopidy_request_data(client, valid_auth_token):
     url = reverse("scrobbles:mopidy-webhook")
@@ -30,55 +40,358 @@ def test_bad_mopidy_request_data(client, valid_auth_token):
     )
 
 
-@pytest.mark.parametrize(
-    "seconds, expected_percent_played, expected_scrobble_id",
-    [
-        (1, 1, 1),
-        (58, 96, 1),
-        (59, 98, 1),
-        (60, 100, 1),
-        (1, 1, 2),
-        (1, 1, 3),
-    ],
-)
 @pytest.mark.django_db
-def test_scrobble_mopidy_track(
+def test_bad_jellyfin_request_data(client, valid_auth_token):
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(url, headers)
+    assert response.status_code == 400
+    assert (
+        response.data["detail"]
+        == "JSON parse error - Expecting value: line 1 column 1 (char 0)"
+    )
+
+
+@pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_create_scrobble_from_mopidy_track_webhook(
+    mock_track_fc,
+    mock_artist_fc,
     client,
-    mopidy_track,
     valid_auth_token,
-    seconds,
-    expected_percent_played,
-    expected_scrobble_id,
+    mopidy_track,
 ):
+    mock_artist = MagicMock(spec=Artist)
+    mock_artist.id = 1
+    mock_artist_fc.return_value = mock_artist
+
+    mock_track = MagicMock(spec=Track)
+    mock_track.id = 1
+    mock_track.scrobble_for_user.return_value = Scrobble(
+        id=1, track_id=1, user_id=1, in_progress=True
+    )
+    mock_track_fc.return_value = mock_track
+
     url = reverse("scrobbles:mopidy-webhook")
     headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(
+        url,
+        mopidy_track.request_data,
+        content_type="application/json",
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 1}
+    mock_track.scrobble_for_user.assert_called_once()
 
-    # Start new scrobble
-    minutes = 0
-    calc_seconds = seconds
-    if seconds >= 60:
-        minutes = 1
-        calc_seconds = calc_seconds % 10
-    with time_machine.travel(datetime(2024, 1, 14, 12, minutes, calc_seconds)):
-        mopidy_track.request_data["playback_time_ticks"] = seconds * 1000
+
+@pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_create_scrobble_from_jellyfin_track_webhook(
+    mock_track_fc,
+    mock_artist_fc,
+    client,
+    valid_auth_token,
+    jellyfin_track,
+):
+    mock_artist = MagicMock(spec=Artist)
+    mock_artist.id = 1
+    mock_artist_fc.return_value = mock_artist
+
+    mock_track = MagicMock(spec=Track)
+    mock_track.id = 1
+    mock_track.scrobble_for_user.return_value = Scrobble(
+        id=1, track_id=1, user_id=1, in_progress=True
+    )
+    mock_track_fc.return_value = mock_track
+
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+
+    with time_machine.travel(datetime(2024, 1, 14, 12, 00, 1)):
+        jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
         response = client.post(
             url,
-            mopidy_track.request_json,
+            jellyfin_track.request_json,
             content_type="application/json",
             headers=headers,
         )
         assert response.status_code == 200
-        assert response.data == {"scrobble_id": expected_scrobble_id}
+        assert response.data == {"scrobble_id": 1}
+        mock_track.scrobble_for_user.assert_called_once()
+
+
+@pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_mopidy_track_webhook_creates_track_and_scrobble(
+    mock_track_fc,
+    mock_artist_fc,
+    client,
+    valid_auth_token,
+    mopidy_track,
+):
+    artist = Artist.objects.create(name="Sublime")
+    album = Album.objects.create(name="Sublime", album_artist=artist)
+    track = Track.objects.create(
+        title="Same in the End",
+        artist=artist,
+        base_run_time_seconds=60,
+    )
+    track.albums.add(album)
+
+    mock_artist_fc.return_value = artist
+    mock_track_fc.return_value = track
+
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(
+        url,
+        mopidy_track.request_data,
+        content_type="application/json",
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 1}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.track == track
+    assert scrobble.source == "Mopidy"
+
+
+@pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_jellyfin_track_webhook_creates_track_and_scrobble(
+    mock_track_fc,
+    mock_artist_fc,
+    client,
+    valid_auth_token,
+    jellyfin_track,
+):
+    artist = Artist.objects.create(name="Carly Rae Jepsen")
+    album = Album.objects.create(name="Emotion", album_artist=artist)
+    track = Track.objects.create(
+        title="Emotion",
+        artist=artist,
+        base_run_time_seconds=60,
+    )
+    track.albums.add(album)
+
+    mock_artist_fc.return_value = artist
+    mock_track_fc.return_value = track
+
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+
+    with time_machine.travel(datetime(2024, 1, 14, 12, 00, 1)):
+        jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
+        response = client.post(
+            url,
+            jellyfin_track.request_json,
+            content_type="application/json",
+            headers=headers,
+        )
+        assert response.status_code == 200
+        assert response.data == {"scrobble_id": 1}
 
         scrobble = Scrobble.objects.get(id=1)
-        assert scrobble.percent_played == expected_percent_played
-        assert scrobble.media_obj.__class__ == Track
-        assert scrobble.media_obj.title == "Same in the End"
+        assert scrobble.track == track
+        assert scrobble.source == "Jellyfin"
+        assert "raw_data" in scrobble.log
 
 
-@pytest.mark.skip(reason="Allmusic API is unstable")
 @pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_mopidy_track_webhook_stores_raw_data(
+    mock_track_fc,
+    mock_artist_fc,
+    client,
+    valid_auth_token,
+    mopidy_track,
+):
+    artist = Artist.objects.create(name="Sublime")
+    album = Album.objects.create(name="Sublime", album_artist=artist)
+    track = Track.objects.create(
+        title="Same in the End",
+        artist=artist,
+        base_run_time_seconds=60,
+    )
+    track.albums.add(album)
+
+    mock_artist_fc.return_value = artist
+    mock_track_fc.return_value = track
+
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(
+        url,
+        mopidy_track.request_data,
+        content_type="application/json",
+        headers=headers,
+    )
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 1}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.track == track
+    assert scrobble.source == "Mopidy"
+    assert "raw_data" in scrobble.log
+    assert scrobble.log["raw_data"]["name"] == "Same in the End"
+
+
+@pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_mopidy_track_webhook_stores_album_id(
+    mock_track_fc,
+    mock_artist_fc,
+    client,
+    valid_auth_token,
+    mopidy_track,
+):
+    artist = Artist.objects.create(name="Sublime")
+    album = Album.objects.create(name="Sublime", album_artist=artist)
+    track = Track.objects.create(
+        title="Same in the End",
+        artist=artist,
+        album=album,
+        base_run_time_seconds=60,
+    )
+
+    mock_artist_fc.return_value = artist
+    mock_track_fc.return_value = track
+
+    url = reverse("scrobbles:mopidy-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    response = client.post(
+        url,
+        mopidy_track.request_data,
+        content_type="application/json",
+        headers=headers,
+    )
+    assert response.status_code == 200
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert "album_id" in scrobble.log
+    assert scrobble.log["album_id"] == album.id
+    assert "album" not in scrobble.log
+
+
+@pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_jellyfin_track_webhook_stores_raw_data(
+    mock_track_fc,
+    mock_artist_fc,
+    client,
+    valid_auth_token,
+    jellyfin_track,
+):
+    artist = Artist.objects.create(name="Carly Rae Jepsen")
+    album = Album.objects.create(name="Emotion", album_artist=artist)
+    track = Track.objects.create(
+        title="Emotion",
+        artist=artist,
+        base_run_time_seconds=60,
+    )
+    track.albums.add(album)
+
+    mock_artist_fc.return_value = artist
+    mock_track_fc.return_value = track
+
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+
+    with time_machine.travel(datetime(2024, 1, 14, 12, 00, 1)):
+        jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
+        response = client.post(
+            url,
+            jellyfin_track.request_json,
+            content_type="application/json",
+            headers=headers,
+        )
+        assert response.status_code == 200
+
+        scrobble = Scrobble.objects.get(id=1)
+        assert scrobble.track == track
+        assert scrobble.source == "Jellyfin"
+        assert "raw_data" in scrobble.log
+        assert scrobble.log["raw_data"]["Name"] == "Emotion"
+
+
+@pytest.mark.django_db
+@patch("music.models.Artist.find_or_create")
+@patch("music.models.Track.find_or_create")
+def test_jellyfin_track_webhook_stores_album_id(
+    mock_track_fc,
+    mock_artist_fc,
+    client,
+    valid_auth_token,
+    jellyfin_track,
+):
+    artist = Artist.objects.create(name="Carly Rae Jepsen")
+    album = Album.objects.create(name="Emotion", album_artist=artist)
+    track = Track.objects.create(
+        title="Emotion",
+        artist=artist,
+        album=album,
+        base_run_time_seconds=60,
+    )
+    track.albums.add(album)
+
+    mock_artist_fc.return_value = artist
+    mock_track_fc.return_value = track
+
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+
+    with time_machine.travel(datetime(2024, 1, 14, 12, 00, 1)):
+        jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
+            "%Y-%m-%d %H:%M:%S"
+        )
+        response = client.post(
+            url,
+            jellyfin_track.request_json,
+            content_type="application/json",
+            headers=headers,
+        )
+        assert response.status_code == 200
+
+        scrobble = Scrobble.objects.get(id=1)
+        assert "album_id" in scrobble.log
+        assert scrobble.log["album_id"] == album.id
+
+
+@pytest.mark.skip("Need to refactor")
+@pytest.mark.django_db
+@patch("music.utils.lookup_artist_from_mb", return_value={})
+@patch(
+    "music.utils.lookup_album_dict_from_mb",
+    return_value={"year": "1999", "mb_group_id": 1},
+)
+@patch("music.utils.lookup_track_from_mb", return_value={})
+@patch("music.models.lookup_artist_from_tadb", return_value={})
+@patch("music.models.lookup_album_from_tadb", return_value={"year": "1999"})
+@patch("music.models.Album.fetch_artwork", return_value=None)
+@patch("music.models.Album.scrape_allmusic", return_value=None)
 def test_scrobble_mopidy_same_track_different_album(
+    mock_lookup_artist,
+    mock_lookup_album,
+    mock_lookup_track,
+    mock_lookup_artist_tadb,
+    mock_lookup_album_tadb,
+    mock_fetch_artwork,
+    mock_scrape_allmusic,
     client,
     mopidy_track,
     mopidy_track_diff_album_request_data,
@@ -107,13 +420,17 @@ def test_scrobble_mopidy_same_track_different_album(
     assert response.data == {"scrobble_id": 2}
     scrobble = Scrobble.objects.last()
     assert scrobble.media_obj.__class__ == Track
-    assert scrobble.media_obj.album.name == "Gold"
+    assert scrobble.media_obj.album.name == "Sublime"
     assert scrobble.media_obj.title == "Same in the End"
 
 
 @pytest.mark.django_db
+@patch(
+    "podcasts.sources.podcastindex.lookup_podcast_from_podcastindex",
+    return_value={},
+)
 def test_scrobble_mopidy_podcast(
-    client, mopidy_podcast_request_data, valid_auth_token
+    mock_lookup_podcast, client, mopidy_podcast_request_data, valid_auth_token
 ):
     url = reverse("scrobbles:mopidy-webhook")
     headers = {"Authorization": f"Token {valid_auth_token}"}
@@ -131,6 +448,7 @@ def test_scrobble_mopidy_podcast(
     assert scrobble.media_obj.title == "Up First"
 
 
+@pytest.mark.skip("Need to refactor")
 @pytest.mark.django_db
 @patch("music.utils.lookup_artist_from_mb", return_value={})
 @patch(
@@ -170,42 +488,223 @@ def test_scrobble_jellyfin_track(
         assert response.status_code == 200
         assert response.data == {"scrobble_id": 1}
 
-        scrobble = Scrobble.objects.get(id=1)
-        assert scrobble.media_obj.__class__ == Track
-        assert scrobble.media_obj.title == "Emotion"
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.__class__ == Track
+    assert scrobble.media_obj.title == "Emotion"
 
-    with time_machine.travel(datetime(2024, 1, 14, 12, 0, 58)):
-        jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
-            "%Y-%m-%d %H:%M:%S"
-        )
-        response = client.post(
-            url,
-            jellyfin_track.request_json,
-            content_type="application/json",
-            headers=headers,
-        )
 
-        assert response.status_code == 200
-        assert response.data == {"scrobble_id": 1}
+@pytest.mark.django_db
+def test_scrobble_detail_view_with_notes_as_flat_list(client):
+    user = get_user_model().objects.create_user(
+        username="testuser", email="test@example.com", password="testpass"
+    )
+    task = Task.objects.create(
+        title="Test Task", description="Test description"
+    )
+    scrobble = Scrobble.objects.create(
+        task=task,
+        media_type="Task",
+        user=user,
+        log={
+            "notes": ["First note", "Second note"],
+            "description": "Test description",
+        },
+    )
+    url = reverse("scrobbles:detail", kwargs={"uuid": scrobble.uuid})
+    response = client.get(url)
+    assert response.status_code == 200
+    assert "First note" in response.content.decode()
+    assert "Second note" in response.content.decode()
 
-        scrobble = Scrobble.objects.get(id=1)
-        assert scrobble.media_obj.__class__ == Track
-        assert scrobble.media_obj.title == "Emotion"
 
-    with time_machine.travel(datetime(2024, 1, 14, 12, 1, 1)):
-        jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
-            "%Y-%m-%d %H:%M:%S"
-        )
-        response = client.post(
-            url,
-            jellyfin_track.request_json,
-            content_type="application/json",
-            headers=headers,
-        )
+@pytest.mark.django_db
+def test_scrobble_detail_view_with_notes_as_dict_timestamps(client):
+    user = get_user_model().objects.create_user(
+        username="testuser", email="test@example.com", password="testpass"
+    )
+    task = Task.objects.create(
+        title="Test Task", description="Test description"
+    )
+    scrobble = Scrobble.objects.create(
+        task=task,
+        media_type="Task",
+        user=user,
+        log={
+            "notes": [
+                {"2024-01-01 10:00:00": "Note at first timestamp"},
+                {"2024-01-02 11:30:00": "Note at second timestamp"},
+            ],
+            "description": "Test description",
+        },
+    )
+    url = reverse("scrobbles:detail", kwargs={"uuid": scrobble.uuid})
+    response = client.get(url)
+    assert response.status_code == 200
+    content = response.content.decode()
+    assert "2024-01-01 10:00:00" in content
+    assert "Note at first timestamp" in content
+    assert "2024-01-02 11:30:00" in content
+    assert "Note at second timestamp" in content
 
-        assert response.status_code == 200
-        assert response.data == {"scrobble_id": 2}
 
-        scrobble = Scrobble.objects.get(id=1)
-        assert scrobble.media_obj.__class__ == Track
-        assert scrobble.media_obj.title == "Emotion"
+@pytest.mark.django_db
+def test_scrobble_detail_view_with_notes_and_labels(client):
+    user = get_user_model().objects.create_user(
+        username="testuser", email="test@example.com", password="testpass"
+    )
+    task = Task.objects.create(
+        title="Test Task", description="Test description"
+    )
+    scrobble = Scrobble.objects.create(
+        task=task,
+        media_type="Task",
+        user=user,
+        log={
+            "notes": [
+                {"2024-01-01 10:00:00": "Note with label"},
+            ],
+            "labels": ["work", "urgent"],
+            "description": "Test description",
+        },
+    )
+    url = reverse("scrobbles:detail", kwargs={"uuid": scrobble.uuid})
+    response = client.get(url)
+    assert response.status_code == 200
+    content = response.content.decode()
+    assert "work" in content
+    assert "urgent" in content
+
+
+@pytest.mark.django_db
+def test_scrobble_detail_view_post_updates_log(client):
+    user = get_user_model().objects.create_user(
+        username="testuser", email="test@example.com", password="testpass"
+    )
+    task = Task.objects.create(
+        title="Test Task", description="Test description"
+    )
+    scrobble = Scrobble.objects.create(
+        task=task,
+        media_type="Task",
+        user=user,
+        log={
+            "notes": ["Original note"],
+            "description": "Original description",
+        },
+    )
+    url = reverse("scrobbles:detail", kwargs={"uuid": scrobble.uuid})
+
+    client.force_login(user)
+    response = client.post(
+        url,
+        {
+            "description": "Updated description",
+            "notes": "Updated note",
+        },
+    )
+    assert response.status_code == 302
+
+    scrobble.refresh_from_db()
+    assert scrobble.log["description"] == "Updated description"
+    assert scrobble.log["notes"] == ["Updated note"]
+
+
+@pytest.mark.skip("Need to refactor")
+@pytest.mark.django_db
+@patch("music.utils.lookup_artist_from_mb", return_value={})
+@patch(
+    "music.utils.lookup_album_dict_from_mb",
+    return_value={"year": "1999", "mb_group_id": 1},
+)
+@patch("music.utils.lookup_track_from_mb", return_value={})
+@patch("music.models.lookup_artist_from_tadb", return_value={})
+@patch("music.models.lookup_album_from_tadb", return_value={"year": "1999"})
+@patch("music.models.Album.fetch_artwork", return_value=None)
+@patch("music.models.Album.scrape_allmusic", return_value=None)
+def test_scrobble_jellyfin_track_update(
+    mock_lookup_artist,
+    mock_lookup_album,
+    mock_lookup_track,
+    mock_lookup_artist_tadb,
+    mock_lookup_album_tadb,
+    mock_fetch_artwork,
+    mock_scrape_allmusic,
+    test_track,
+    client,
+    jellyfin_track,
+    valid_auth_token,
+):
+    Scrobble.objects.create(
+        timestamp=timezone.now() - timedelta(minutes=0.5),
+        track=Track.objects.first(),
+        user_id=1,
+    )
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+
+    jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+    response = client.post(
+        url,
+        jellyfin_track.request_json,
+        content_type="application/json",
+        headers=headers,
+    )
+
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 1}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.__class__ == Track
+    assert scrobble.media_obj.title == "Emotion"
+
+
+@pytest.mark.skip("Need to refactor")
+@pytest.mark.django_db
+@patch("music.utils.lookup_artist_from_mb", return_value={})
+@patch(
+    "music.utils.lookup_album_dict_from_mb",
+    return_value={"year": "1999", "mb_group_id": 1},
+)
+@patch("music.utils.lookup_track_from_mb", return_value={})
+@patch("music.models.lookup_artist_from_tadb", return_value={})
+@patch("music.models.lookup_album_from_tadb", return_value={"year": "1999"})
+@patch("music.models.Album.fetch_artwork", return_value=None)
+@patch("music.models.Album.scrape_allmusic", return_value=None)
+def test_scrobble_jellyfin_track_create_new(
+    mock_lookup_artist,
+    mock_lookup_album,
+    mock_lookup_track,
+    mock_lookup_artist_tadb,
+    mock_lookup_album_tadb,
+    mock_fetch_artwork,
+    mock_scrape_allmusic,
+    test_track,
+    client,
+    jellyfin_track,
+    valid_auth_token,
+):
+    url = reverse("scrobbles:jellyfin-webhook")
+    headers = {"Authorization": f"Token {valid_auth_token}"}
+    Scrobble.objects.create(
+        timestamp=timezone.now() - timedelta(minutes=1),
+        track=Track.objects.first(),
+        user_id=1,
+    )
+    jellyfin_track.request_data["UtcTimestamp"] = timezone.now().strftime(
+        "%Y-%m-%d %H:%M:%S"
+    )
+    response = client.post(
+        url,
+        jellyfin_track.request_json,
+        content_type="application/json",
+        headers=headers,
+    )
+
+    assert response.status_code == 200
+    assert response.data == {"scrobble_id": 2}
+
+    scrobble = Scrobble.objects.get(id=1)
+    assert scrobble.media_obj.__class__ == Track
+    assert scrobble.media_obj.title == "Emotion"

tests/test_context_processors.py

@@ -0,0 +1,95 @@
+import os
+from unittest.mock import patch, MagicMock
+from pathlib import Path
+
+import pytest
+from vrobbler.context_processors import version_info
+
+
+@pytest.fixture
+def mock_request():
+    return MagicMock()
+
+
+class TestVersionInfo:
+    def test_returns_version_and_commit(self, mock_request):
+        with (
+            patch(
+                "vrobbler.context_processors.get_version"
+            ) as mock_get_version,
+            patch(
+                "vrobbler.context_processors.subprocess.check_output"
+            ) as mock_check_output,
+        ):
+            mock_get_version.return_value = "1.0.0"
+            mock_check_output.return_value = b"abc1234"
+
+            result = version_info(mock_request)
+
+            assert result["app_version"] == "1.0.0"
+            assert result["git_commit"] == "abc1234"
+
+    def test_uses_env_commit_if_set(self, mock_request):
+        with (
+            patch.dict(os.environ, {"VROBBLER_COMMIT": "env_commit_hash"}),
+            patch(
+                "vrobbler.context_processors.get_version"
+            ) as mock_get_version,
+        ):
+            mock_get_version.return_value = "1.0.0"
+
+            result = version_info(mock_request)
+
+            assert result["git_commit"] == "env_commit_hash"
+
+    def test_returns_unknown_when_version_fails(self, mock_request):
+        with (
+            patch(
+                "vrobbler.context_processors.get_version"
+            ) as mock_get_version,
+            patch(
+                "vrobbler.context_processors.subprocess.check_output"
+            ) as mock_check_output,
+        ):
+            mock_get_version.side_effect = Exception("not found")
+            mock_check_output.return_value = b"abc1234"
+
+            result = version_info(mock_request)
+
+            assert result["app_version"] == "unknown"
+
+    def test_returns_unknown_when_git_fails(self, mock_request):
+        import subprocess
+
+        with (
+            patch(
+                "vrobbler.context_processors.get_version"
+            ) as mock_get_version,
+            patch(
+                "vrobbler.context_processors.subprocess.check_output"
+            ) as mock_check_output,
+        ):
+            mock_get_version.return_value = "1.0.0"
+            mock_check_output.side_effect = subprocess.SubprocessError()
+
+            result = version_info(mock_request)
+
+            assert result["git_commit"] == "unknown"
+
+    def test_returns_unknown_when_git_not_found(self, mock_request):
+        import subprocess
+
+        with (
+            patch(
+                "vrobbler.context_processors.get_version"
+            ) as mock_get_version,
+            patch(
+                "vrobbler.context_processors.subprocess.check_output"
+            ) as mock_check_output,
+        ):
+            mock_get_version.return_value = "1.0.0"
+            mock_check_output.side_effect = FileNotFoundError()
+
+            result = version_info(mock_request)
+
+            assert result["git_commit"] == "unknown"

tests/videos_tests/test_api.py

@@ -0,0 +1,104 @@
+import pytest
+from django.contrib.auth import get_user_model
+from rest_framework.authtoken.models import Token
+
+from videos.models import Channel, Series, Video
+
+User = get_user_model()
+
+
+@pytest.fixture
+def auth_headers():
+    user = User.objects.create(email="api@test.com")
+    token = Token.objects.create(user=user)
+    return {"HTTP_AUTHORIZATION": f"Token {token.key}"}
+
+
+@pytest.fixture
+def channel():
+    return Channel.objects.create(name="Test Channel")
+
+
+@pytest.fixture
+def series():
+    return Series.objects.create(
+        name="Test Series",
+    )
+
+
+@pytest.fixture
+def video(channel):
+    return Video.objects.create(
+        title="Test Video",
+        imdb_id="tt1234567",
+        channel=channel,
+    )
+
+
+@pytest.mark.django_db
+class TestVideoAPI:
+    def test_list_videos(self, client, auth_headers, video):
+        response = client.get("/api/v1/videos/", **auth_headers)
+        assert response.status_code == 200
+        assert len(response.data["results"]) == 1
+        assert response.data["results"][0]["title"] == "Test Video"
+
+    def test_get_video(self, client, auth_headers, video):
+        response = client.get(f"/api/v1/videos/{video.id}/", **auth_headers)
+        assert response.status_code == 200
+        assert response.data["title"] == "Test Video"
+
+    def test_filter_videos_by_channel(
+        self, client, auth_headers, channel, video
+    ):
+        response = client.get(
+            f"/api/v1/videos/?channel={channel.id}", **auth_headers
+        )
+        assert response.status_code == 200
+        assert len(response.data["results"]) == 1
+        assert response.data["results"][0]["channel"] == channel.id
+
+
+@pytest.mark.django_db
+class TestChannelAPI:
+    def test_list_channels(self, client, auth_headers, channel):
+        response = client.get("/api/v1/channels/", **auth_headers)
+        assert response.status_code == 200
+        assert len(response.data["results"]) == 1
+        assert response.data["results"][0]["name"] == "Test Channel"
+
+    def test_get_channel(self, client, auth_headers, channel):
+        response = client.get(
+            f"/api/v1/channels/{channel.id}/", **auth_headers
+        )
+        assert response.status_code == 200
+        assert response.data["name"] == "Test Channel"
+
+
+@pytest.mark.django_db
+class TestSeriesAPI:
+    def test_list_series(self, client, auth_headers, series):
+        response = client.get("/api/v1/series/", **auth_headers)
+        assert response.status_code == 200
+        assert len(response.data["results"]) == 1
+        assert response.data["results"][0]["name"] == "Test Series"
+
+    def test_get_series(self, client, auth_headers, series):
+        response = client.get(f"/api/v1/series/{series.id}/", **auth_headers)
+        assert response.status_code == 200
+        assert response.data["name"] == "Test Series"
+
+
+@pytest.mark.django_db
+class TestVideoAPIUnauthorized:
+    def test_list_videos_unauthenticated(self, client, video):
+        response = client.get("/api/v1/videos/")
+        assert response.status_code == 401
+
+    def test_list_channels_unauthenticated(self, client, channel):
+        response = client.get("/api/v1/channels/")
+        assert response.status_code == 401
+
+    def test_list_series_unauthenticated(self, client, series):
+        response = client.get("/api/v1/series/")
+        assert response.status_code == 401

tests/videos_tests/test_imdb.py

@@ -1,11 +1,14 @@
 import pytest
-
-from videos.imdb import lookup_video_from_imdb
+from videos.sources.imdb import lookup_video_from_imdb
 
 
-@pytest.mark.skip(reason="Need to sort out third party API testing")
-def test_lookup_imdb_bad_id(caplog):
-    data = lookup_video_from_imdb("3409324")
-    assert data is None
-    assert caplog.records[0].levelname == "WARNING"
-    assert caplog.records[0].msg == "IMDB ID should begin with 'tt' 3409324"
+@pytest.mark.skip(reason="Should not hit IMDB api in CI")
+def test_lookup_imdb_without_tt():
+    metadata = lookup_video_from_imdb("8946378")
+    assert not metadata.imdb_id
+
+
+@pytest.mark.skip(reason="Should not hit IMDB api in CI")
+def test_lookup_imdb_with_tt():
+    metadata = lookup_video_from_imdb("tt8946378")
+    assert metadata.title == "Knives Out"

tests/videos_tests/test_youtube.py

@@ -0,0 +1,9 @@
+import pytest
+from videos.sources.youtube import lookup_video_from_youtube
+
+
+@pytest.mark.skip(reason="Need to configure Youtube API stuffs in CI")
+@pytest.mark.django_db
+def test_lookup_youtube_id():
+    metadata = lookup_video_from_youtube("RZxs9pAv99Y")
+    assert metadata.title == "No Pun Included's Board Game of the Year 2024"

todos.org

@@ -1,449 +0,0 @@
-#+title: TODOs
-
-A fun way to keep track of things in the project to fix or improve.
-
-* Version 1.0.0
-** TODO What to do with Youtube videos from LastFM and web-scrobbler :bug:source:lastfm:
-** TODO Add a user profile page with ability to change settings :profiles:improvement:
-** DONE [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :utililty:improvement:
-CLOSED: [2023-04-06 Thu 14:09]
-** DONE Add a "stop_timestamp" so we don't rely on content length :improvement:scrobbling:
-CLOSED: [2023-04-02 Sun 23:58]
-
-Essentially, we currently have the timestamp as when the content began
-scrobbling and then calculate the finish time from the length of the content.
-This works pretty well because we know how long most things are.
-
-But in some cases, sports events or long podcasts, we may start mid-way through
-an event or finish halfway through but still want to mark it as done. In these
-cases, knowing the finish time could be useful, especially when interfacing with
-other scrobblers which may have different definitions of when a scrobble
-finishes or started.
-** DONE Fix bug with Various Artist albums being labeled with first artist as album artist :scrobbling:bug:music:
-CLOSED: [2023-03-27 Mon 20:18]
-:LOGBOOK:
-CLOCK: [2023-03-26 Sun 22:01]--[2023-03-27 Mon 01:07] =>  3:06
-:END:
-** DONE Fix bug with weekly aggregator being blank on Sundays :aggregators:music:bug:
-CLOSED: [2023-03-26 Sun 13:52]
-** DONE Fix KoReader scrobbling to use pages rather than time of last read :scrobbling:books:improvement:
-CLOSED: [2023-03-26 Sun 13:51]
-:LOGBOOK:
-CLOCK: [2023-03-26 Sun 13:11]--[2023-03-26 Sun 13:51] =>  0:40
-:END:
-** DONE [#A] Add django-storage to store files on S3 :settings:improvement:
-CLOSED: [2023-03-24 Fri 14:46]
-:LOGBOOK:
-CLOCK: [2023-03-24 Fri 10:47]--[2023-03-24 Fri 14:46] =>  3:59
-CLOCK: [2023-03-24 Fri 10:36]--[2023-03-24 Fri 10:40] =>  0:04
-:END:
-** DONE Fix vrobbler settings not using booleans :settings:bug:
-CLOSED: [2023-03-24 Fri 10:45]
-:LOGBOOK:
-CLOCK: [2023-03-24 Fri 10:40]--[2023-03-24 Fri 10:46] =>  0:06
-:END:
-** DONE Update weekly live chart to be 7-day continuous rather than weekly :views:bug:
-CLOSED: [2023-03-24 Fri 00:31]
-The live view will be blank every Monday, no reason to tie it to a day of the
-week. It should be "the last 7 days"
-** DONE [#B] Implement a detail view for TV shows :improvement:views:
-CLOSED: [2023-03-22 Wed 17:05]
-** DONE [#B] Implement a detail view for Movies :improvement:views:
-CLOSED: [2023-03-22 Wed 17:05]
-** DONE Add "service provider" to TV Series, and use that for source when available :bug:scrobbling:
-CLOSED: [2023-03-22 Wed 17:04]
-** DONE Add view for long-play content (books, video games) to restart them :views:improvement:
-CLOSED: [2023-03-22 Wed 17:01]
-** DONE Add live chart view like Maloja :improvement:views:
-CLOSED: [2023-03-07 Tue 11:13]
-** DONE [#C] Figure out how to add to web-scrobbler :improvement:scrobbling:
-CLOSED: [2023-03-22 Wed 17:06]
-
-An example:
-https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
-
-This is actually going to be moot because we can import from LastFM, and
-web-scrobbler integrates well with LastFM. The only thing to think through here
-now is what to do with all the garbage web-scrobbler sometimes pushes to LastFM
-from Youtube (all videos get pushed, sigh).
-
-* Version 0.11.4
-** DONE Add rudimentary video game scrobbling :improvement:content:videogames:
-CLOSED: [2023-03-07 Tue 11:11]
-** DONE Add ability to scrobble from KOReader statistics files :improvement:books:content:
-CLOSED: [2023-03-07 Tue 11:11]
-
-** DONE [#A] Fix fetching artwork without release group :bug:
-CLOSED: [2023-01-29 Sun 14:27]
-
-When we get artwork from Musicbrianz, and it's not found, we should check for
-release groups as well. This will stop issues with missing artwork because of
-obscure MB release matches.
-
-** DONE [#A] Fix Jellyfin music scrobbling N+1 past 90 completion percent :bug:
-CLOSED: [2023-01-30 Mon 18:31]
-:LOGBOOK:
-CLOCK: [2023-01-30 Mon 18:00]--[2023-01-30 Mon 18:31] =>  0:31
-:END:
-
-If we play music from Jellyfin and the track reaches 90% completion, the
-scrobbling goes crazy and starts creating new scrobbles with every update.
-
-The cause is pretty simple, but the solution is hard. We want to mark a scrobble
-as complete for the following conditions:
-
-- Play stopped and percent played beyond 90%
-- Play completely finished
-
-But if we keep listening beyond 90, we should basically ignore updates (or just
-update the existing scrobble)
-** DONE [#A] Add support for Audioscrobbler tab-separated file uploads :improvement:
-CLOSED: [2023-02-03 Fri 16:52]
-
-An example of the format:
-#+begin_src csv
-,
-#AUDIOSCROBBLER/1.1
-#TZ/UNKNOWN
-#CLIENT/Rockbox sansaclipplus $Revision$
-75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494944	64ff5f53-d187-4512-827e-7606c69e66ff
-75 Dollar Bill	I Was Real	I Was Real	4	1015	S	1740494990	64ff5f53-d187-4512-827e-7606c69e66ff
-311	311	Down	1	173	S	1740495003	00476c23-fd9e-464b-9b27-a62d69f3d4f4
-311	311	Down	1	173	L	1740495049	00476c23-fd9e-464b-9b27-a62d69f3d4f4
-311	311	Down	1	173	L	1740495113	00476c23-fd9e-464b-9b27-a62d69f3d4f4
-311	311	Random	2	187	S	1740495190	530c09f3-46fe-4d90-b11f-7b63bcb4b373
-311	311	Random	2	187	L	1740495194	530c09f3-46fe-4d90-b11f-7b63bcb4b373
-311	311	Jackolantern’s Weather	3	204	L	1740495382	cc3b2dec-5d99-47ea-8930-20bf258be4ea
-311	311	All Mixed Up	4	182	L	1740495586	980a78b5-5bdd-4f50-9e3a-e13261e2817b
-311	311	Hive	5	179	L	1740495768	18f6dc98-d3a2-4f81-b967-97359d14c68c
-311	311	Guns (Are for Pussies)	6	137	L	1740495948	5e97ed9f-c8cc-4282-9cbe-f8e17aee5128
-311	311	Misdirected Hostility	7	179	S	1740496085	61ff2c1a-fc9c-44c3-8da1-5e50a44245af
-,
-#+end_src
-** DONE [#B] Allow scrobbling music without MB IDs by grabbing them before scrobble :improvement:
-CLOSED: [2023-02-17 Fri 00:10]
-
-This would allow a few nice flows. One, you'd be able to record the play of an
-entire album by just dropping the muscibrainz_id in. This could be helpful for
-offline listening. It would also mean bad metadata from mopidy would not break
-scrobbling.
-** DONE When updating musicbrainz IDs, clear and run fetch artwrok :improvement:
-CLOSED: [2023-02-17 Fri 00:11]
-** DONE [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
-CLOSED: [2023-03-07 Tue 11:09]
-
-Given a UUID from musicbrainz, we should be able to scrobble an album or
-individual track.
-
-** DONE [#C] Implement keeping track of week/month/year chart-toppers :improvement:
-CLOSED: [2023-03-07 Tue 11:10]
-:LOGBOOK:
-CLOCK: [2023-01-30 Mon 16:30]--[2023-01-30 Mon 18:00] =>  1:30
-:END:
-
-Maloja does this cool thing where artists and tracks get recorded as the top
-track of a given week, month or year. They get gold, silver or bronze stars for
-their place in the time period.
-
-I could see this being implemented as a separate Chart table which gets
-populated at the end of a time period and has a start and end date that defines
-a period, along with a one, two, three instance.
-
-Of course, it could also be a data model without a table, where it runs some fun
-calculations, stores it's values in Redis as a long-term lookup table and just
-has to re-populate when the server restarts.
-* Backlog
-** TODO Add Amazon scraper to look up books when OL fails :books:improvement:
-** TODO [#C] Move to using more robust mopidy-webhooks pacakge form pypi :utility:improvement:
-*** Example payloads from mopidy-webhooks
-**** Podcast playback ended
-#+begin_src json
-{
-    "type": "event",
-    "event": "track_playback_ended",
-    "data": {
-        "tl_track": {
-            "__model__": "TlTrack",
-            "tlid": 13,
-            "track": {
-                "__model__": "Track",
-                "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
-                "name": "Wolf warriors",
-                "artists": [
-                    {
-                        "__model__": "Artist",
-                        "name": "The Economist"
-                    }
-                ],
-                "album": {
-                    "__model__": "Album",
-                    "name": "The Prince",
-                    "date": "2022"
-                },
-                "genre": "Blues",
-                "date": "2022",
-                "length": 2437778,
-                "bitrate": 127988
-            }
-        },
-        "time_position": 3290
-    }
-}
-#+end_src
-**** Podcast playback state changes
-#+begin_src json
-{
-    "type": "event",
-    "event": "playback_state_changed",
-    "data": {
-        "old_state": "paused",
-        "new_state": "playing"
-    }
-}
-#+end_src
-
-#+begin_src json
-{
-    "type": "event",
-    "event": "playback_state_changed",
-    "data": {
-        "old_state": "stopped",
-        "new_state": "playing"
-    }
-}
-#+end_src
-**** Podcast playback started
-#+begin_src json
-{
-    "type": "event",
-    "event": "track_playback_started",
-    "data": {
-        "tl_track": {
-            "__model__": "TlTrack",
-            "tlid": 13,
-            "track": {
-                "__model__": "Track",
-                "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
-                "name": "Wolf warriors",
-                "artists": [
-                    {
-                        "__model__": "Artist",
-                        "name": "The Economist"
-                    }
-                ],
-                "album": {
-                    "__model__": "Album",
-                    "name": "The Prince",
-                    "date": "2022"
-                },
-                "genre": "Blues",
-                "date": "2022",
-                "length": 2437778,
-                "bitrate": 127988
-            }
-        }
-    }
-}
-#+end_src
-**** Podcast playback paused
-#+begin_src  json
-{
-    "type": "status",
-    "data": {
-        "state": "paused",
-        "current_track": {
-            "__model__": "Track",
-            "uri": "file:///var/lib/mopidy/media/podcasts/The%20Prince/2022-09-28-Wolf-warriors.mp3",
-            "name": "Wolf warriors",
-            "artists": [
-                {
-                    "__model__": "Artist",
-                    "name": "The Economist"
-                }
-            ],
-            "album": {
-                "__model__": "Album",
-                "name": "The Prince",
-                "date": "2022"
-            },
-            "genre": "Blues",
-            "date": "2022",
-            "length": 2437778,
-            "bitrate": 127988
-        },
-        "time_position": 2350
-    }
-}
-
-#+end_src
-**** Track playback started
-#+begin_src json
-{
-    "type": "event",
-    "event": "track_playback_started",
-    "data": {
-        "tl_track": {
-            "__model__": "TlTrack",
-            "tlid": 14,
-            "track": {
-                "__model__": "Track",
-                "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
-                "name": "Supermassive Black Hole",
-                "artists": [
-                    {
-                        "__model__": "Artist",
-                        "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
-                        "name": "Muse",
-                        "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
-                    }
-                ],
-                "album": {
-                    "__model__": "Album",
-                    "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
-                    "name": "Twilight: Original Motion Picture Soundtrack",
-                    "artists": [
-                        {
-                            "__model__": "Artist",
-                            "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
-                            "name": "Various Artists",
-                            "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
-                        }
-                    ],
-                    "num_tracks": 12,
-                    "num_discs": 1,
-                    "date": "2008-11-04",
-                    "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
-                },
-                "composers": [
-                    {
-                        "__model__": "Artist",
-                        "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
-                        "name": "Matt Bellamy"
-                    }
-                ],
-                "genre": "Rock",
-                "track_no": 1,
-                "disc_no": 1,
-                "date": "2008-11-04",
-                "length": 211121,
-                "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
-                "last_modified": 1672712949510
-            }
-        }
-    }
-}
-#+end_src
-**** Track playback in progress
-#+begin_src json
-{
-    "type": "status",
-    "data": {
-        "state": "playing",
-        "current_track": {
-            "__model__": "Track",
-            "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
-            "name": "Supermassive Black Hole",
-            "artists": [
-                {
-                    "__model__": "Artist",
-                    "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
-                    "name": "Muse",
-                    "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
-                }
-            ],
-            "album": {
-                "__model__": "Album",
-                "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
-                "name": "Twilight: Original Motion Picture Soundtrack",
-                "artists": [
-                    {
-                        "__model__": "Artist",
-                        "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
-                        "name": "Various Artists",
-                        "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
-                    }
-                ],
-                "num_tracks": 12,
-                "num_discs": 1,
-                "date": "2008-11-04",
-                "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
-            },
-            "composers": [
-                {
-                    "__model__": "Artist",
-                    "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
-                    "name": "Matt Bellamy"
-                }
-            ],
-            "genre": "Rock",
-            "track_no": 1,
-            "disc_no": 1,
-            "date": "2008-11-04",
-            "length": 211121,
-            "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
-            "last_modified": 1672712949510
-        },
-        "time_position": 17031
-    }
-}
-#+end_src
-**** Track event playback paused
-#+begin_src json
-{
-    "type": "event",
-    "event": "track_playback_paused",
-    "data": {
-        "tl_track": {
-            "__model__": "TlTrack",
-            "tlid": 14,
-            "track": {
-                "__model__": "Track",
-                "uri": "local:track:Various%20Artists%20-%202008%20-%20Twilight%20OST/01-muse-supermassive_black_hole.mp3",
-                "name": "Supermassive Black Hole",
-                "artists": [
-                    {
-                        "__model__": "Artist",
-                        "uri": "local:artist:md5:250dd6551b66a58a6b4897aa697f200c",
-                        "name": "Muse",
-                        "musicbrainz_id": "9c9f1380-2516-4fc9-a3e6-f9f61941d090"
-                    }
-                ],
-                "album": {
-                    "__model__": "Album",
-                    "uri": "local:album:md5:455343d54cdd89cb5a3b5ad537ea99d0",
-                    "name": "Twilight: Original Motion Picture Soundtrack",
-                    "artists": [
-                        {
-                            "__model__": "Artist",
-                            "uri": "local:artist:md5:54e4db2d5624f80b0cc290346e696756",
-                            "name": "Various Artists",
-                            "musicbrainz_id": "89ad4ac3-39f7-470e-963a-56509c546377"
-                        }
-                    ],
-                    "num_tracks": 12,
-                    "num_discs": 1,
-                    "date": "2008-11-04",
-                    "musicbrainz_id": "b4889eaf-d9f4-434c-a68d-69227b12b6a4"
-                },
-                "composers": [
-                    {
-                        "__model__": "Artist",
-                        "uri": "local:artist:md5:4d49cbca0b347e0a89047bb019d2779d",
-                        "name": "Matt Bellamy"
-                    }
-                ],
-                "genre": "Rock",
-                "track_no": 1,
-                "disc_no": 1,
-                "date": "2008-11-04",
-                "length": 211121,
-                "musicbrainz_id": "ff1e3e1a-f6e8-4692-b426-355880383bb6",
-                "last_modified": 1672712949510
-            }
-        },
-        "time_position": 67578
-    }
-}
-#+end_src
-** TODO Fix bug in Jellyfin scrobbles that spam more scrobbles after completion :scrobbling:videos:bug:
-** TODO Fix bug in podcast scrobbling where a second scrobble is created after completion :scrobbling:podcasts:bug:

vrobbler.conf.example

@@ -21,6 +21,10 @@ VROBBLER_THEAUDIODB_API_KEY="<key>"
 VROBBLER_IGDB_CLIENT_ID="<id>"
 VROBBLER_IGDB_CLIENT_SECRET="<key>"
 VROBBLER_COMICVINE_API_KEY="<key>"
+VROBBLER_TODOIST_CLIENT_ID="<id>"
+VROBBLER_TODOIST_CLIENT_SECRET="<key>"
+VROBBLER_GOOGLE_API_KEY="<key>"
+VROBBLER_LICHESS_API_KEY = "<key>"
 
 # Storages
 # VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"

vrobbler/apps/beers/admin.py

@@ -0,0 +1,34 @@
+from beers.models import Beer, BeerProducer, BeerStyle
+from django.contrib import admin
+from scrobbles.admin import ScrobbleInline
+
+
+class BeerInline(admin.TabularInline):
+    model = Beer
+    extra = 0
+
+
+@admin.register(BeerStyle)
+class BeerStyle(admin.ModelAdmin):
+    date_hierarchy = "created"
+    search_fields = ("name",)
+
+
+@admin.register(BeerProducer)
+class BeerProducer(admin.ModelAdmin):
+    date_hierarchy = "created"
+    search_fields = ("name",)
+
+
+@admin.register(Beer)
+class BeerAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "uuid",
+        "title",
+    )
+    ordering = ("-created",)
+    search_fields = ("title",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/beers/api/serializers.py

@@ -0,0 +1,20 @@
+from rest_framework import serializers
+from beers.models import Beer, BeerProducer, BeerStyle
+
+
+class BeerSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = Beer
+        fields = "__all__"
+
+
+class BeerProducerSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = BeerProducer
+        fields = "__all__"
+
+
+class BeerStyleSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = BeerStyle
+        fields = "__all__"

vrobbler/apps/beers/api/views.py

@@ -0,0 +1,21 @@
+from rest_framework import permissions, viewsets
+from beers.api import serializers
+from beers import models
+
+
+class BeerViewSet(viewsets.ModelViewSet):
+    queryset = models.Beer.objects.all().order_by("-created")
+    serializer_class = serializers.BeerSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class BeerProducerViewSet(viewsets.ModelViewSet):
+    queryset = models.BeerProducer.objects.all().order_by("-created")
+    serializer_class = serializers.BeerProducerSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class BeerStyleViewSet(viewsets.ModelViewSet):
+    queryset = models.BeerStyle.objects.all().order_by("-created")
+    serializer_class = serializers.BeerStyleSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/beers/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class BeersConfig(AppConfig):
+    name = "beers"

vrobbler/apps/beers/migrations/0001_initial.py

@@ -0,0 +1,133 @@
+# Generated by Django 4.2.16 on 2024-10-22 21:26
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import taggit.managers
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ("scrobbles", "0065_alter_scrobble_log"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="BeerProducer",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("description", models.TextField(blank=True, null=True)),
+                (
+                    "location",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+        migrations.CreateModel(
+            name="Beer",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    "title",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "run_time_seconds",
+                    models.IntegerField(blank=True, null=True),
+                ),
+                (
+                    "run_time_ticks",
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                ("description", models.TextField(blank=True, null=True)),
+                ("ibu", models.SmallIntegerField(blank=True, null=True)),
+                ("abv", models.FloatField(blank=True, null=True)),
+                (
+                    "style",
+                    models.CharField(blank=True, max_length=100, null=True),
+                ),
+                ("non_alcoholic", models.BooleanField(default=False)),
+                (
+                    "beeradvocate_id",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "beeradvocate_score",
+                    models.SmallIntegerField(blank=True, null=True),
+                ),
+                (
+                    "untappd_id",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "genre",
+                    taggit.managers.TaggableManager(
+                        blank=True,
+                        help_text="A comma-separated list of tags.",
+                        through="scrobbles.ObjectWithGenres",
+                        to="scrobbles.Genre",
+                        verbose_name="Tags",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

vrobbler/apps/beers/migrations/0002_beer_beeradvocate_image_beer_producer_and_more.py

@@ -0,0 +1,36 @@
+# Generated by Django 4.2.16 on 2024-10-22 21:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("beers", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="beer",
+            name="beeradvocate_image",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="beers/beeradvcoate/"
+            ),
+        ),
+        migrations.AddField(
+            model_name="beer",
+            name="producer",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="beers.beerproducer",
+            ),
+        ),
+        migrations.AddField(
+            model_name="beerproducer",
+            name="beeradvocate_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/beers/migrations/0003_beerstyle_remove_beer_beeradvocate_image_and_more.py

@@ -0,0 +1,75 @@
+# Generated by Django 4.2.16 on 2024-10-22 21:47
+
+from django.db import migrations, models
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("beers", "0002_beer_beeradvocate_image_beer_producer_and_more"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="BeerStyle",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("description", models.TextField(blank=True, null=True)),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+        migrations.RemoveField(
+            model_name="beer",
+            name="beeradvocate_image",
+        ),
+        migrations.RemoveField(
+            model_name="beer",
+            name="style",
+        ),
+        migrations.AddField(
+            model_name="beer",
+            name="untappd_image",
+            field=models.ImageField(
+                blank=True, null=True, upload_to="beers/untappd/"
+            ),
+        ),
+        migrations.AddField(
+            model_name="beer",
+            name="untappd_rating",
+            field=models.FloatField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="beerproducer",
+            name="untappd_id",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="beer",
+            name="styles",
+            field=models.ManyToManyField(to="beers.beerstyle"),
+        ),
+    ]

vrobbler/apps/beers/migrations/0004_beerproducer_name_beerproducer_uuid_beerstyle_name_and_more.py

@@ -0,0 +1,47 @@
+# Generated by Django 4.2.16 on 2024-10-22 21:52
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("beers", "0003_beerstyle_remove_beer_beeradvocate_image_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="beerproducer",
+            name="name",
+            field=models.CharField(default="Untitled", max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="beerproducer",
+            name="uuid",
+            field=models.UUIDField(
+                blank=True, default=uuid.uuid4, editable=False, null=True
+            ),
+        ),
+        migrations.AddField(
+            model_name="beerstyle",
+            name="name",
+            field=models.CharField(default="Untitled", max_length=255),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name="beerstyle",
+            name="uuid",
+            field=models.UUIDField(
+                blank=True, default=uuid.uuid4, editable=False, null=True
+            ),
+        ),
+        migrations.AlterField(
+            model_name="beer",
+            name="styles",
+            field=models.ManyToManyField(
+                related_name="styles", to="beers.beerstyle"
+            ),
+        ),
+    ]

vrobbler/apps/beers/migrations/0005_alter_beer_run_time_seconds.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.16 on 2025-01-22 03:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "beers",
+            "0004_beerproducer_name_beerproducer_uuid_beerstyle_name_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="beer",
+            name="run_time_seconds",
+            field=models.IntegerField(default=900),
+        ),
+    ]

vrobbler/apps/beers/migrations/0006_remove_beer_run_time_seconds_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.19 on 2025-10-30 01:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('beers', '0005_alter_beer_run_time_seconds'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='beer',
+            name='run_time_seconds',
+        ),
+        migrations.RemoveField(
+            model_name='beer',
+            name='run_time_ticks',
+        ),
+        migrations.AddField(
+            model_name='beer',
+            name='base_run_time_seconds',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/beers/models.py

@@ -0,0 +1,147 @@
+from dataclasses import dataclass
+from typing import Optional
+from uuid import uuid4
+
+from beers.untappd import get_beer_from_untappd_id
+from django.apps import apps
+from django.db import models
+from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
+from imagekit.models import ImageSpecField
+from imagekit.processors import ResizeToFit
+from scrobbles.dataclasses import BaseLogData
+from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
+
+BNULL = {"blank": True, "null": True}
+
+
+@dataclass
+class BeerLogData(BaseLogData):
+    rating: Optional[str] = None
+
+
+class BeerStyle(TimeStampedModel):
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    name = models.CharField(max_length=255)
+    description = models.TextField(**BNULL)
+
+    def __str__(self):
+        return self.name
+
+
+class BeerProducer(TimeStampedModel):
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    name = models.CharField(max_length=255)
+    description = models.TextField(**BNULL)
+    location = models.CharField(max_length=255, **BNULL)
+    beeradvocate_id = models.CharField(max_length=255, **BNULL)
+    untappd_id = models.CharField(max_length=255, **BNULL)
+
+    def find_or_create(cls, title: str) -> "BeerProducer":
+        return cls.objects.filter(title=title).first()
+
+    def __str__(self):
+        return self.name
+
+
+class Beer(ScrobblableMixin):
+    description = models.TextField(**BNULL)
+    ibu = models.SmallIntegerField(**BNULL)
+    abv = models.FloatField(**BNULL)
+    styles = models.ManyToManyField(BeerStyle, related_name="styles")
+    non_alcoholic = models.BooleanField(default=False)
+    beeradvocate_id = models.CharField(max_length=255, **BNULL)
+    beeradvocate_score = models.SmallIntegerField(**BNULL)
+    untappd_image = models.ImageField(upload_to="beers/untappd/", **BNULL)
+    untappd_image_small = ImageSpecField(
+        source="untappd_image",
+        processors=[ResizeToFit(100, 100)],
+        format="JPEG",
+        options={"quality": 60},
+    )
+    untappd_image_medium = ImageSpecField(
+        source="untappd_image",
+        processors=[ResizeToFit(300, 300)],
+        format="JPEG",
+        options={"quality": 75},
+    )
+    untappd_id = models.CharField(max_length=255, **BNULL)
+    untappd_rating = models.FloatField(**BNULL)
+    producer = models.ForeignKey(
+        BeerProducer, on_delete=models.DO_NOTHING, **BNULL
+    )
+
+    def get_absolute_url(self) -> str:
+        return reverse("beers:beer_detail", kwargs={"slug": self.uuid})
+
+    def __str__(self):
+        return f"{self.title} by {self.producer}"
+
+    @property
+    def subtitle(self):
+        return self.producer.name
+
+    @property
+    def strings(self) -> ScrobblableConstants:
+        return ScrobblableConstants(verb="Drinking", tags="beer")
+
+    @property
+    def beeradvocate_link(self) -> str:
+        link = ""
+        if self.producer and self.beeradvocate_id:
+            if self.beeradvocate_id:
+                link = f"https://www.beeradvocate.com/beer/profile/{self.producer.beeradvocate_id}/{self.beeradvocate_id}/"
+        return link
+
+    @property
+    def untappd_link(self) -> str:
+        link = ""
+        if self.untappd_id:
+            link = f"https://www.untappd.com/beer/{self.untappd_id}/"
+        return link
+
+    @property
+    def primary_image_url(self) -> str:
+        url = ""
+        if self.untappd_image:
+            url = self.untappd_image.url
+        return url
+
+    @property
+    def logdata_cls(self):
+        return BeerLogData
+
+    @classmethod
+    def find_or_create(cls, untappd_id: str) -> "Beer":
+        beer = cls.objects.filter(untappd_id=untappd_id).first()
+
+        if not beer:
+            beer_dict = get_beer_from_untappd_id(untappd_id)
+            producer_dict = {}
+            style_ids = []
+            for key in list(beer_dict.keys()):
+                if "producer__" in key:
+                    pkey = key.replace("producer__", "")
+                    producer_dict[pkey] = beer_dict.pop(key)
+                if "styles" in key:
+                    for style in beer_dict.pop("styles"):
+                        style_inst, created = BeerStyle.objects.get_or_create(
+                            name=style
+                        )
+                        style_ids.append(style_inst.id)
+
+            producer, _created = BeerProducer.objects.get_or_create(
+                **producer_dict
+            )
+            beer_dict["producer_id"] = producer.id
+            beer = Beer.objects.create(**beer_dict)
+            for style_id in style_ids:
+                beer.styles.add(style_id)
+
+        return beer
+
+    def scrobbles(self, user_id):
+        Scrobble = apps.get_model("scrobbles", "Scrobble")
+        return Scrobble.objects.filter(user_id=user_id, beer=self).order_by(
+            "-timestamp"
+        )

vrobbler/apps/beers/untappd.py

@@ -0,0 +1,142 @@
+import logging
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+UNTAPPD_URL = "https://untappd.com/beer/{id}"
+
+
+def get_first(key: str, result: dict) -> str:
+    obj = ""
+    if obj_list := result.get(key):
+        obj = obj_list[0]
+    return obj
+
+
+def get_title_from_soup(soup) -> str:
+    title = ""
+    try:
+        title = soup.find("h1").get_text()
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return title
+
+
+def get_description_from_soup(soup) -> str:
+    desc = ""
+    try:
+        desc = (
+            soup.find(class_="beer-descrption-read-less")
+            .get_text()
+            .replace("Show Less", "")
+            .strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return desc
+
+
+def get_styles_from_soup(soup) -> list[str]:
+    styles = []
+    try:
+        styles = soup.find("p", class_="style").get_text().split(" - ")
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return styles
+
+
+def get_abv_from_soup(soup) -> Optional[float]:
+    abv = None
+    try:
+        abv = soup.find(class_="abv").get_text()
+        if abv:
+            abv = float(abv.strip("\n").strip("% ABV").strip())
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except TypeError:
+        pass
+    return abv
+
+
+def get_ibu_from_soup(soup) -> Optional[int]:
+    ibu = None
+    try:
+        ibu = soup.find(class_="ibu").get_text()
+        if ibu:
+            ibu = int(ibu.strip("\n").strip(" IBU").strip())
+    except AttributeError:
+        pass
+    except ValueError:
+        ibu = None
+    return ibu
+
+
+def get_rating_from_soup(soup) -> str:
+    rating = ""
+    try:
+        rating = float(
+            soup.find(class_="num").get_text().strip("(").strip(")")
+        )
+    except AttributeError:
+        rating = None
+    except ValueError:
+        rating = None
+    return rating
+
+
+def get_producer_id_from_soup(soup) -> str:
+    id = ""
+    try:
+        id = soup.find(class_="brewery").find("a")["href"].strip("/")
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return id
+
+
+def get_producer_name_from_soup(soup) -> str:
+    name = ""
+    try:
+        name = soup.find(class_="brewery").find("a").get_text()
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return name
+
+
+def get_beer_from_untappd_id(untappd_id: str) -> dict:
+    beer_url = UNTAPPD_URL.format(id=untappd_id)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    response = requests.get(beer_url, headers=headers)
+    beer_dict = {"untappd_id": untappd_id}
+
+    if response.status_code != 200:
+        logger.warn(
+            "Bad response from untappd.com", extra={"response": response}
+        )
+        return beer_dict
+
+    soup = BeautifulSoup(response.text, "html.parser")
+    beer_dict["title"] = get_title_from_soup(soup)
+    beer_dict["description"] = get_description_from_soup(soup)
+    beer_dict["styles"] = get_styles_from_soup(soup)
+    beer_dict["abv"] = get_abv_from_soup(soup)
+    beer_dict["ibu"] = get_ibu_from_soup(soup)
+    beer_dict["untappd_rating"] = get_rating_from_soup(soup)
+    beer_dict["producer__untappd_id"] = get_producer_id_from_soup(soup)
+    beer_dict["producer__name"] = get_producer_name_from_soup(soup)
+
+    return beer_dict

vrobbler/apps/beers/urls.py

@@ -0,0 +1,14 @@
+from django.urls import path
+from beers import views
+
+app_name = "beers"
+
+
+urlpatterns = [
+    path("beers/", views.BeerListView.as_view(), name="beer_list"),
+    path(
+        "beers/<slug:slug>/",
+        views.BeerDetailView.as_view(),
+        name="beer_detail",
+    ),
+]

vrobbler/apps/beers/views.py

@@ -0,0 +1,11 @@
+from beers.models import Beer
+
+from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView
+
+
+class BeerListView(ScrobbleableListView):
+    model = Beer
+
+
+class BeerDetailView(ScrobbleableDetailView):
+    model = Beer

vrobbler/apps/boardgames/admin.py

@@ -1,6 +1,11 @@
 from django.contrib import admin
 
-from boardgames.models import BoardGame, BoardGamePublisher
+from boardgames.models import (
+    BoardGame,
+    BoardGameLocation,
+    BoardGamePublisher,
+    BoardGameDesigner,
+)
 
 from scrobbles.admin import ScrobbleInline
 
@@ -15,13 +20,34 @@ class BoardGamePublisherAdmin(admin.ModelAdmin):
     ordering = ("-created",)
 
 
+@admin.register(BoardGameDesigner)
+class BoardGameDesignerAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "name",
+        "uuid",
+    )
+    ordering = ("-created",)
+
+
+@admin.register(BoardGameLocation)
+class BoardGameLocationAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "name",
+        "uuid",
+        "geo_location",
+    )
+    ordering = ("-created",)
+
+
 @admin.register(BoardGame)
-class GameAdmin(admin.ModelAdmin):
+class BoardGameAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     list_display = (
         "bggeek_id",
         "title",
-        "published_date",
+        "published_year",
     )
     search_fields = ("title",)
     ordering = ("-created",)

vrobbler/apps/boardgames/api/serializers.py

@@ -0,0 +1,26 @@
+from boardgames import models
+from rest_framework import serializers
+
+
+class BoardGameDesignerSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = models.BoardGameDesigner
+        fields = "__all__"
+
+
+class BoardGamePublisherSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = models.BoardGamePublisher
+        fields = "__all__"
+
+
+class BoardGameLocationSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = models.BoardGameLocation
+        fields = "__all__"
+
+
+class BoardGameSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = models.BoardGame
+        fields = "__all__"

vrobbler/apps/boardgames/api/views.py

@@ -0,0 +1,28 @@
+from rest_framework import permissions, viewsets
+
+from boardgames.api import serializers
+from boardgames import models
+
+
+class BoardGameDesignerViewSet(viewsets.ModelViewSet):
+    queryset = models.BoardGameDesigner.objects.all().order_by("-created")
+    serializer_class = serializers.BoardGameDesignerSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class BoardGamePublisherViewSet(viewsets.ModelViewSet):
+    queryset = models.BoardGamePublisher.objects.all().order_by("-created")
+    serializer_class = serializers.BoardGamePublisherSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class BoardGameLocationViewSet(viewsets.ModelViewSet):
+    queryset = models.BoardGameLocation.objects.all().order_by("-created")
+    serializer_class = serializers.BoardGameLocationSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+
+class BoardGameViewSet(viewsets.ModelViewSet):
+    queryset = models.BoardGame.objects.all().order_by("-created")
+    serializer_class = serializers.BoardGameSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/boardgames/bgg.py

@@ -1,8 +1,16 @@
+import csv
+import json
 import logging
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 import requests
 from bs4 import BeautifulSoup
+from django.contrib.auth import get_user_model
+from django.conf import settings
+
+User = get_user_model()
+if TYPE_CHECKING:
+    from scrobbles.models import Scrobble
 
 logger = logging.getLogger(__name__)
 
@@ -10,6 +18,11 @@ SEARCH_ID_URL = (
     "https://boardgamegeek.com/xmlapi/search?search={query}&exact=1"
 )
 GAME_ID_URL = "https://boardgamegeek.com/xmlapi/boardgame/{id}"
+BGG_ACCESS_TOKEN = getattr(settings, "BGG_ACCESS_TOKEN", "")
+BASE_HEADERS = {
+    "User-Agent": "Vrobbler 31.0",
+    "Authorization": f"Bearer {BGG_ACCESS_TOKEN}",
+}
 
 
 def take_first(thing: Optional[list]) -> str:
@@ -30,10 +43,9 @@ def take_first(thing: Optional[list]) -> str:
 
 def lookup_boardgame_id_from_bgg(title: str) -> Optional[int]:
     soup = None
-    headers = {"User-Agent": "Vrobbler 0.11.12"}
     game_id = None
     url = SEARCH_ID_URL.format(query=title)
-    r = requests.get(url, headers=headers)
+    r = requests.get(url, headers=BASE_HEADERS)
     if r.status_code == 200:
         soup = BeautifulSoup(r.text, "xml")
 
@@ -50,7 +62,6 @@ def lookup_boardgame_id_from_bgg(title: str) -> Optional[int]:
 def lookup_boardgame_from_bgg(lookup_id: str) -> dict:
     soup = None
     game_dict = {}
-    headers = {"User-Agent": "Vrobbler 0.11.12"}
 
     title = ""
     bgg_id = None
@@ -66,7 +77,7 @@ def lookup_boardgame_from_bgg(lookup_id: str) -> dict:
         bgg_id = lookup_boardgame_id_from_bgg(title)
 
     url = GAME_ID_URL.format(id=bgg_id)
-    r = requests.get(url, headers=headers)
+    r = requests.get(url, headers=BASE_HEADERS)
     if r.status_code == 200:
         soup = BeautifulSoup(r.text, "xml")
 
@@ -90,3 +101,58 @@ def lookup_boardgame_from_bgg(lookup_id: str) -> dict:
         }
 
     return game_dict
+
+
+def push_scrobble_to_bgg(scrobble: "Scrobble", user: User) -> Optional[bool]:
+    bgg_username = "secstate"  # user.profile.bgg_username
+    bgg_password = "yYFCKnfo8AK89lc68q0S"
+
+    if not bgg_username or bgg_password:
+        return
+
+    login_payload = {
+        "credentials": {"username": bgg_username, "password": bgg_password}
+    }
+    headers = BASE_HEADERS
+    headers["content-type"] = "application/json"
+
+    # TODO Look up past plays for scrobble.media_obj.bggeek_id, and make sure we haven't scrobbled this before
+
+    with requests.Session() as s:
+        p = s.post(
+            "https://boardgamegeek.com/login/api/v1",
+            data=json.dumps(login_payload),
+            headers=headers,
+        )
+        players = []
+        if scrobble.log:
+            for player in scrobble.log.get("players"):
+                player_person = Person.objects.filter(
+                    id=player.get("person_id")
+                ).first()
+                if player_person.get("bgg_username"):
+                    player["username"] = player_person.get("bgg_username")
+                player["name"] = player_person.get("name")
+                player["win"] = player.get("win")
+                # player["role"] = player.get("role")
+                player["new"] = player.get("new")
+                player["score"] = player.get("score")
+                players.append(player)
+
+        play_payload = {
+            "playdate": scrobble.timestamp.date.strftime("%Y-%m-%d"),
+            "length": scrobble.playback_position_seconds / 60,
+            "comments": "Uploaded from Vrobbler",
+            "location": scrobble.log.location or None,
+            "objectid": scrobble.media_obj.bggeek_id,
+            "quantity": "1",
+            "action": "save",
+            "players": players,
+            "objecttype": "thing",
+            "ajax": 1,
+        }
+        r = s.post(
+            "https://boardgamegeek.com/geekplay.php",
+            data=json.dumps(play_payload),
+            headers=headers,
+        )

vrobbler/apps/boardgames/migrations/0007_alter_boardgame_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2025-01-22 03:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0006_alter_boardgame_genre"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="boardgame",
+            name="run_time_seconds",
+            field=models.IntegerField(default=900),
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0008_boardgamedesigner_and_more.py

@@ -0,0 +1,167 @@
+# Generated by Django 4.2.19 on 2025-07-03 01:57
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("locations", "0007_alter_geolocation_run_time_seconds"),
+        ("boardgames", "0007_alter_boardgame_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="BoardGameDesigner",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("name", models.CharField(max_length=255)),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ("bgg_id", models.IntegerField(blank=True, null=True)),
+                ("bio", models.TextField(blank=True, null=True)),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+        migrations.RenameField(
+            model_name="boardgamepublisher",
+            old_name="igdb_id",
+            new_name="bgg_id",
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="bgstats_id",
+            field=models.UUIDField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="cooperative",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="expansion_for_boardgame",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to="boardgames.boardgame",
+            ),
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="highest_wins",
+            field=models.BooleanField(default=True),
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="max_play_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="min_play_time",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="no_points",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="uses_teams",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.CreateModel(
+            name="BoardGameLocation",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                ("name", models.CharField(max_length=255)),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ("bgstats_id", models.UUIDField(blank=True, null=True)),
+                ("description", models.TextField(blank=True, null=True)),
+                (
+                    "geo_location",
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to="locations.geolocation",
+                    ),
+                ),
+            ],
+            options={
+                "get_latest_by": "modified",
+                "abstract": False,
+            },
+        ),
+        migrations.AddField(
+            model_name="boardgame",
+            name="designers",
+            field=models.ManyToManyField(
+                related_name="board_games", to="boardgames.boardgamedesigner"
+            ),
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0009_alter_boardgame_cooperative_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.19 on 2025-07-03 02:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0008_boardgamedesigner_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="boardgame",
+            name="cooperative",
+            field=models.BooleanField(blank=True, default=False, null=True),
+        ),
+        migrations.AlterField(
+            model_name="boardgame",
+            name="highest_wins",
+            field=models.BooleanField(blank=True, default=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="boardgame",
+            name="no_points",
+            field=models.BooleanField(blank=True, default=False, null=True),
+        ),
+        migrations.AlterField(
+            model_name="boardgame",
+            name="uses_teams",
+            field=models.BooleanField(blank=True, default=False, null=True),
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0010_boardgame_published_year.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-07-03 04:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("boardgames", "0009_alter_boardgame_cooperative_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="boardgame",
+            name="published_year",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0011_remove_boardgame_run_time_seconds_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.19 on 2025-10-30 01:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boardgames', '0010_boardgame_published_year'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='boardgame',
+            name='run_time_seconds',
+        ),
+        migrations.RemoveField(
+            model_name='boardgame',
+            name='run_time_ticks',
+        ),
+        migrations.AddField(
+            model_name='boardgame',
+            name='base_run_time_seconds',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0012_boardgame_bgg_rank.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-11-03 04:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boardgames', '0011_remove_boardgame_run_time_seconds_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='boardgame',
+            name='bgg_rank',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/boardgames/migrations/0013_boardgame_publishers.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-11-03 04:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('boardgames', '0012_boardgame_bgg_rank'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='boardgame',
+            name='publishers',
+            field=models.ManyToManyField(related_name='board_games', to='boardgames.boardgamepublisher'),
+        ),
+    ]

vrobbler/apps/boardgames/models.py

@@ -1,10 +1,13 @@
 import logging
+from dataclasses import dataclass
 from datetime import datetime
-from typing import Optional
+from functools import cached_property
+from typing import Any, Optional
 from uuid import uuid4
 
 import requests
-from boardgames.bgg import lookup_boardgame_from_bgg
+from boardgames.sources.bgg import lookup_boardgame_from_bgg
+from django import forms
 from django.conf import settings
 from django.core.files.base import ContentFile
 from django.db import models
@@ -12,18 +15,120 @@ from django.urls import reverse
 from django_extensions.db.models import TimeStampedModel
 from imagekit.models import ImageSpecField
 from imagekit.processors import ResizeToFit
-from scrobbles.dataclasses import BoardGameLogData
-from scrobbles.mixins import ScrobblableMixin
+from locations.models import GeoLocation
+from people.models import Person
+from scrobbles.dataclasses import BaseLogData, LongPlayLogData
+from scrobbles.mixins import ScrobblableConstants, ScrobblableMixin
 
 logger = logging.getLogger(__name__)
 BNULL = {"blank": True, "null": True}
 
 
+@dataclass
+class BoardGameScoreLogData(BaseLogData):
+    person_id: Optional[int] = None
+    bgg_username: Optional[str] = None
+    color: Optional[str] = None
+    character: Optional[str] = None
+    team: Optional[str] = None
+    score: Optional[int] = None
+    win: Optional[bool] = None
+    new: Optional[bool] = None
+    rank: Optional[int] = None
+    seat_order: Optional[int] = None
+    role: Optional[str] = None
+    rank: Optional[int] = None
+    seat_order: Optional[int] = None
+    role: Optional[str] = None
+    lichess_username: Optional[str] = None
+
+    @property
+    def person(self) -> Optional[Person]:
+        return Person.objects.filter(id=self.person_id).first()
+
+    @property
+    def name(self) -> str:
+        name = ""
+        if self.person:
+            name = self.person.name
+        return name
+
+    def __str__(self) -> str:
+        out = self.name
+        if self.score:
+            out += f" {self.score}"
+        if self.color:
+            out += f" ({self.color})"
+        if self.win:
+            out += f" [W]"
+        return out
+
+
+@dataclass
+class BoardGameLogData(BaseLogData, LongPlayLogData):
+    players: Optional[list[BoardGameScoreLogData]] = None
+    location_id: Optional[int] = None
+    difficulty: Optional[int] = None
+    solo: Optional[bool] = None
+    two_handed: Optional[bool] = None
+    expansion_ids: Optional[int] = None
+    moves: Optional[list] = None
+    rated: Optional[str] = None
+    speed: Optional[str] = None
+    variant: Optional[str] = None
+    lichess_id: Optional[int] = None
+    board: Optional[str] = None
+    rounds: Optional[int] = None
+    details: Optional[str] = None
+
+    _excluded_fields = {
+        "lichess_id",
+        "speed",
+        "rated",
+        "moves",
+        "variant",
+    }
+
+    @cached_property
+    def location(self):
+        if not self.location_id:
+            return
+        return BoardGameLocation.objects.filter(id=self.location_id).first()
+
+    @cached_property
+    def player_log(self) -> str:
+        if self.players:
+            return ", ".join(
+                [
+                    BoardGameScoreLogData(**player).__str__()
+                    for player in self.players
+                ]
+            )
+        return ""
+
+    @classmethod
+    def override_fields(cls) -> dict:
+        fields = {}
+        for base in cls.mro()[1:]:
+            if hasattr(base, "override_fields"):
+                base_fields = base.override_fields()
+                fields.update(base_fields)
+        custom_fields = {
+            "location_id": forms.ModelChoiceField(
+                queryset=BoardGameLocation.objects.all(),
+                required=False,
+                widget=forms.Select(),
+            )
+        }
+        fields.update(custom_fields)
+        return fields
+
+
 class BoardGamePublisher(TimeStampedModel):
     name = models.CharField(max_length=255)
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     logo = models.ImageField(upload_to="games/platform-logos/", **BNULL)
-    igdb_id = models.IntegerField(**BNULL)
+    bgg_id = models.IntegerField(**BNULL)
 
     def __str__(self):
         return self.name
@@ -34,6 +139,39 @@ class BoardGamePublisher(TimeStampedModel):
         )
 
 
+class BoardGameDesigner(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    bgg_id = models.IntegerField(**BNULL)
+    bio = models.TextField(**BNULL)
+
+    def __str__(self) -> str:
+        return str(self.name)
+
+    def get_absolute_url(self):
+        return reverse(
+            "boardgames:designer_detail", kwargs={"slug": self.uuid}
+        )
+
+
+class BoardGameLocation(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
+    bgstats_id = models.UUIDField(**BNULL)
+    description = models.TextField(**BNULL)
+    geo_location = models.ForeignKey(
+        GeoLocation, **BNULL, on_delete=models.DO_NOTHING
+    )
+
+    def __str__(self) -> str:
+        return str(self.name)
+
+    def get_absolute_url(self):
+        return reverse(
+            "boardgames:location_detail", kwargs={"slug": self.uuid}
+        )
+
+
 class BoardGame(ScrobblableMixin):
     COMPLETION_PERCENT = getattr(
         settings, "BOARD_GAME_COMPLETION_PERCENT", 100
@@ -53,6 +191,14 @@ class BoardGame(ScrobblableMixin):
     publisher = models.ForeignKey(
         BoardGamePublisher, **BNULL, on_delete=models.DO_NOTHING
     )
+    publishers = models.ManyToManyField(
+        BoardGamePublisher,
+        related_name="board_games",
+    )
+    designers = models.ManyToManyField(
+        BoardGameDesigner,
+        related_name="board_games",
+    )
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
     description = models.TextField(**BNULL)
     cover = models.ImageField(upload_to="boardgames/covers/", **BNULL)
@@ -82,11 +228,23 @@ class BoardGame(ScrobblableMixin):
         options={"quality": 75},
     )
     rating = models.FloatField(**BNULL)
+    bgg_rank = models.IntegerField(**BNULL)
     max_players = models.PositiveSmallIntegerField(**BNULL)
     min_players = models.PositiveSmallIntegerField(**BNULL)
     published_date = models.DateField(**BNULL)
+    published_year = models.IntegerField(**BNULL)
     recommended_age = models.PositiveSmallIntegerField(**BNULL)
     bggeek_id = models.CharField(max_length=255, **BNULL)
+    bgstats_id = models.UUIDField(**BNULL)
+    uses_teams = models.BooleanField(default=False, **BNULL)
+    cooperative = models.BooleanField(default=False, **BNULL)
+    highest_wins = models.BooleanField(default=True, **BNULL)
+    no_points = models.BooleanField(default=False, **BNULL)
+    min_play_time = models.IntegerField(**BNULL)
+    max_play_time = models.IntegerField(**BNULL)
+    expansion_for_boardgame = models.ForeignKey(
+        "self", **BNULL, on_delete=models.DO_NOTHING
+    )
 
     def __str__(self):
         return self.title
@@ -100,6 +258,10 @@ class BoardGame(ScrobblableMixin):
     def logdata_cls(self):
         return BoardGameLogData
 
+    @property
+    def strings(self) -> ScrobblableConstants:
+        return ScrobblableConstants(verb="Playing", tags="game_die")
+
     def primary_image_url(self) -> str:
         url = ""
         if self.cover:
@@ -112,9 +274,6 @@ class BoardGame(ScrobblableMixin):
             link = f"https://boardgamegeek.com/boardgame/{self.bggeek_id}"
         return link
 
-    def get_start_url(self):
-        return reverse("scrobbles:start", kwargs={"uuid": self.uuid})
-
     def fix_metadata(self, data: dict = {}, force_update=False) -> None:
 
         if not self.published_date or force_update:
@@ -127,7 +286,12 @@ class BoardGame(ScrobblableMixin):
             publisher_name = data.pop("publisher_name")
 
             if year:
-                data["published_date"] = datetime(int(year), 1, 1)
+                data["published_year"] = int(year)
+
+            if not data["min_players"]:
+                data.pop("min_players")
+            if not data["min_players"]:
+                data.pop("max_players")
 
             # Fun trick for updating all fields at once
             BoardGame.objects.filter(pk=self.id).update(**data)
@@ -142,29 +306,72 @@ class BoardGame(ScrobblableMixin):
 
             # Go get cover image if the URL is present
             if cover_url and not self.cover:
-                headers = {"User-Agent": "Vrobbler 0.11.12"}
-                r = requests.get(cover_url, headers=headers)
-                logger.debug(r.status_code)
-                if r.status_code == 200:
-                    fname = f"{self.title}_cover_{self.uuid}.jpg"
-                    self.cover.save(fname, ContentFile(r.content), save=True)
-                    logger.debug("Loaded cover image from BGGeek")
+                self.save_image_from_url(cover_url)
+
+    def save_image_from_url(self, url):
+        headers = {"User-Agent": "Vrobbler 0.11.12"}
+        r = requests.get(url, headers=headers)
+        if r.status_code == 200:
+            fname = f"{self.title}_cover_{self.uuid}.jpg"
+            self.cover.save(fname, ContentFile(r.content), save=True)
 
     @classmethod
     def find_or_create(
-        cls, lookup_id: str, data: Optional[dict] = {}
-    ) -> Optional["BoardGame"]:
+        cls, lookup_id: str, data: dict[str, Any] = {}
+    ) -> "BoardGame":
         """Given a Lookup ID (either BGG or BGA ID), return a board game object"""
-        boardgame = cls.objects.filter(bggeek_id=lookup_id).first()
+        game = cls.objects.filter(bggeek_id=lookup_id).first()
+        if not game:
+            game = cls.objects.filter(title=lookup_id).first()
 
-        if not data or not boardgame:
-            data = lookup_boardgame_from_bgg(lookup_id)
-
-        if data and not boardgame:
-            boardgame, created = cls.objects.get_or_create(
-                title=data["title"], bggeek_id=lookup_id
+        if game:
+            logger.info(
+                "Board game exists in database.",
+                extra={"lookup_id": lookup_id, "data": data},
             )
-            if created:
-                boardgame.fix_metadata(data=data)
+            return game
 
-        return boardgame
+        if data.get("bggId"):
+            bgg_data = lookup_boardgame_from_bgg(lookup_id=data.get("bggId"))
+        elif data.get("name"):
+            bgg_data = lookup_boardgame_from_bgg(title=data.get("name"))
+        else:
+            bgg_data = lookup_boardgame_from_bgg(title=lookup_id)
+
+        mechanics = bgg_data.pop("mechanics", [])
+        designers = bgg_data.pop("designers", [])
+        categories = bgg_data.pop("categories", [])
+        publishers = bgg_data.pop("publishers", [])
+        publisher = bgg_data.pop("publisher", [])
+        cover_url = bgg_data.pop("cover_url")
+
+        game = cls.objects.create(**bgg_data)
+
+        game.save_image_from_url(cover_url)
+        game.cooperative = data.get("cooperative", False)
+        game.highest_wins = data.get("highestWins", True)
+        game.no_points = data.get("noPoints", False)
+        game.uses_teams = data.get("useTeams", False)
+        game.bgstats_id = data.get("uuid", None)
+        if publisher:
+            publisher, _ = BoardGamePublisher.objects.get_or_create(
+                name=publisher
+            )
+            game.publisher = publisher
+        game.save()
+
+        if designers:
+            for designer_name in designers:
+                designer, created = BoardGameDesigner.objects.get_or_create(
+                    name=designer_name
+                )
+                game.designers.add(designer.id)
+
+        if publishers:
+            for name in publishers:
+                publisher, _ = BoardGamePublisher.objects.get_or_create(
+                    name=name
+                )
+                game.publishers.add(publisher)
+
+        return game

vrobbler/apps/boardgames/sources/bgg.py

@@ -0,0 +1,40 @@
+from typing import Any, Union
+from boardgamegeek import BGGClient
+
+from django.conf import settings
+
+
+def lookup_boardgame_from_bgg(
+    lookup_id: str | None = None, title: str | None = None
+) -> dict[str, Any]:
+    game_dict: dict[str, Any] = {}
+
+    bgg = BGGClient(access_token=settings.BGG_ACCESS_TOKEN)
+
+    if lookup_id:
+        game = bgg.game(game_id=lookup_id)
+    else:
+        game = bgg.game(title)
+
+    if game:
+        game_dict["title"] = game.name
+        game_dict["description"] = game.description
+        game_dict["published_year"] = game.yearpublished
+        game_dict["cover_url"] = game.image
+        game_dict["min_players"] = game.minplayers
+        game_dict["max_players"] = game.maxplayers
+        game_dict["recommended_age"] = game.minage
+        game_dict["rating"] = game.rating_average
+        game_dict["bggeek_id"] = game.id
+        game_dict["bgg_rank"] = game.bgg_rank
+        game_dict["base_run_time_seconds"] = (
+            int(game.playingtime) * 60 if game.playingtime else None
+        )
+        game_dict["mechanics"] = game.mechanics
+        game_dict["categories"] = game.categories
+        game_dict["designers"] = game.designers
+        game_dict["publishers"] = game.publishers
+        if game.publishers:
+            game_dict["publisher"] = game.publishers[0]
+
+    return game_dict

vrobbler/apps/boardgames/sources/lichess.py

@@ -0,0 +1,128 @@
+import berserk
+from boardgames.models import BoardGame
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from scrobbles.models import Scrobble
+from scrobbles.notifications import ScrobbleNtfyNotification
+
+User = get_user_model()
+
+
+def import_chess_games_for_user_id(user_id: int, commit: bool = False) -> dict:
+    user = User.objects.get(id=user_id)
+
+    client = berserk.Client(
+        session=berserk.TokenSession(settings.LICHESS_API_KEY)
+    )
+    games = client.games.export_by_player(user.profile.lichess_username)
+    for game_dict in games:
+        chess, created = BoardGame.objects.get_or_create(title="Chess")
+        if created:
+            chess.base_run_time_seconds = 1800
+            chess.bggeek_id = 171
+            chess.save(update_fields=["base_run_time_seconds", "bggeek_id"])
+        scrobble = Scrobble.objects.filter(
+            user_id=user.id,
+            timestamp=game_dict.get("createdAt"),
+            board_game_id=chess.id,
+        ).first()
+
+        if scrobble:
+            continue
+
+        log_data = {
+            "variant": game_dict.get("variant"),
+            "lichess_id": game_dict.get("id"),
+            "rated": game_dict.get("rated"),
+            "speed": game_dict.get("speed"),
+            "moves": game_dict.get("moves"),
+            "players": [],
+        }
+
+        winner = game_dict.get("winner")
+        black_player = game_dict.get("players", {}).get("black", {})
+        white_player = game_dict.get("players", {}).get("white", {})
+
+        user_player = {
+            "user_id": user.id,
+            "lichess_username": user.profile.lichess_username,
+            "bgg_username": user.profile.bgg_username,
+            "color": "",
+            "win": False,
+        }
+        other_player = {"name_str": "", "color": "", "win": False}
+
+        if (
+            black_player.get("user", {}).get("name", "")
+            == user.profile.lichess_username
+        ):
+            user_player["color"] = "black"
+            if "aiLevel" in white_player.keys():
+                other_player["name_str"] = "aiLevel_" + str(
+                    white_player.get("aiLevel", "")
+                )
+            else:
+                other_player["name_str"] = white_player.get("user", {}).get(
+                    "name", ""
+                )
+                other_player["lichess_username"] = other_player["name_str"]
+
+            other_player["color"] = "white"
+            if winner == "black":
+                user_player["win"] = True
+            else:
+                other_player["win"] = True
+        if (
+            white_player.get("user", {}).get("name", "")
+            == user.profile.lichess_username
+        ):
+            user_player["color"] = "white"
+            if "aiLevel" in black_player.keys():
+                other_player["name_str"] = "aiLevel_" + str(
+                    black_player.get("aiLevel", "")
+                )
+            else:
+                other_player["name_str"] = black_player.get("user", {}).get(
+                    "name", ""
+                )
+                other_player["lichess_username"] = other_player["name_str"]
+            other_player["color"] = "black"
+            if winner == "white":
+                user_player["win"] = True
+            else:
+                other_player["win"] = True
+
+        log_data["players"].append(user_player)
+        log_data["players"].append(other_player)
+
+        scrobble_dict = {
+            "media_type": Scrobble.MediaType.BOARD_GAME,
+            "user_id": user.id,
+            "playback_position_seconds": (
+                game_dict.get("lastMoveAt") - game_dict.get("createdAt")
+            ).seconds,
+            "in_progress": False,
+            "played_to_completion": True,
+            "timestamp": game_dict.get("createdAt"),
+            "stop_timestamp": game_dict.get("lastMoveAt"),
+            "board_game_id": chess.id,
+            "source": "Lichess",
+            "timezone": user.profile.timezone,
+            "log": log_data,
+        }
+        if commit:
+            Scrobble.objects.create(**scrobble_dict)
+        return scrobble_dict
+
+
+def import_chess_games_for_all_users():
+    scrobbles_to_create = []
+    for user in User.objects.filter(profile__lichess_username__isnull=False):
+        scrobble_dict = import_chess_games_for_user_id(user.id)
+        scrobbles_to_create.append(Scrobble(**scrobble_dict))
+
+    if scrobbles_to_create:
+        created = Scrobble.objects.bulk_create(scrobbles_to_create)
+        for scrobble in created:
+            ScrobbleNtfyNotification(scrobble).send()
+    return scrobbles_to_create

vrobbler/apps/boardgames/urls.py

@@ -6,10 +6,12 @@ app_name = "boardgames"
 
 urlpatterns = [
     path(
-        "board-game/", views.BoardGameListView.as_view(), name="boardgame_list"
+        "board-games/",
+        views.BoardGameListView.as_view(),
+        name="boardgame_list",
     ),
     path(
-        "board-game/<slug:slug>/",
+        "board-games/<slug:slug>/",
         views.BoardGameDetailView.as_view(),
         name="boardgame_detail",
     ),

vrobbler/apps/boardgames/views.py

@@ -1,15 +1,71 @@
+import datetime
+from django.utils import timezone
 from django.views import generic
-from boardgames.models import BoardGame, BoardGamePublisher
+from boardgames.models import BoardGame, BoardGameDesigner, BoardGamePublisher
+from scrobbles.models import Scrobble
+from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView
 
 
-class BoardGameListView(generic.ListView):
+class BoardGameListView(ScrobbleableListView):
     model = BoardGame
-    paginate_by = 20
+
+    def get_context_data(self, **kwargs):
+        context_data = super().get_context_data(**kwargs)
+        user = self.request.user
+        now = timezone.now()
+        start_day_of_week = now - datetime.timedelta(days=now.weekday())
+        start_day_of_month = now.replace(day=1)
+
+        scrobbles_this_week = Scrobble.objects.filter(
+            user=user,
+            board_game__isnull=False,
+            timestamp__gte=start_day_of_week,
+        ).select_related("board_game")
+
+        scrobbles_this_month = Scrobble.objects.filter(
+            user=user,
+            board_game__isnull=False,
+            timestamp__gte=start_day_of_month,
+        ).select_related("board_game")
+
+        designers_this_week = {}
+        for scrobble in scrobbles_this_week:
+            for designer in scrobble.board_game.designers.all():
+                designers_this_week[designer.id] = {
+                    "designer": designer,
+                    "count": designers_this_week.get(designer.id, {}).get(
+                        "count", 0
+                    )
+                    + 1,
+                }
+
+        designers_this_month = {}
+        for scrobble in scrobbles_this_month:
+            for designer in scrobble.board_game.designers.all():
+                designers_this_month[designer.id] = {
+                    "designer": designer,
+                    "count": designers_this_month.get(designer.id, {}).get(
+                        "count", 0
+                    )
+                    + 1,
+                }
+
+        context_data["designers_this_week"] = sorted(
+            designers_this_week.values(),
+            key=lambda x: x["count"],
+            reverse=True,
+        )
+        context_data["designers_this_month"] = sorted(
+            designers_this_month.values(),
+            key=lambda x: x["count"],
+            reverse=True,
+        )
+
+        return context_data
 
 
-class BoardGameDetailView(generic.DetailView):
+class BoardGameDetailView(ScrobbleableDetailView):
     model = BoardGame
-    slug_field = "uuid"
 
 
 class BoardGamePublisherDetailView(generic.DetailView):

vrobbler/apps/books/admin.py

@@ -1,7 +1,5 @@
+from books.models import Author, Book, Paper
 from django.contrib import admin
-
-from books.models import Author, Book, Page
-
 from scrobbles.admin import ScrobbleInline
 
 
@@ -18,22 +16,36 @@ class AuthorAdmin(admin.ModelAdmin):
     search_fields = ("name",)
 
 
-@admin.register(Page)
-class PageAdmin(admin.ModelAdmin):
-    date_hierarchy = "created"
-    list_filter = ("book",)
-    ordering = ("book", "number")
-
-
 @admin.register(Book)
 class BookAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     list_display = (
         "title",
-        "isbn",
+        "author",
+        "issue_or_volume",
+        "isbn_13",
+        "first_publish_year",
+        "pages",
+    )
+    search_fields = ("name",)
+    ordering = ("-created",)
+    inlines = [
+        ScrobbleInline,
+    ]
+
+    def issue_or_volume(self, obj):
+        return obj.issue_number or obj.volume_number
+
+
+@admin.register(Paper)
+class BookAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "title",
+        "subtitle",
+        "arxiv_id",
         "first_publish_year",
         "pages",
-        "openlibrary_id",
     )
     search_fields = ("name",)
     ordering = ("-created",)

vrobbler/apps/books/api/views.py

@@ -1,19 +1,16 @@
 from rest_framework import permissions, viewsets
 
-from books.api.serializers import (
-    AuthorSerializer,
-    BookSerializer,
-)
-from books.models import Author, Book
+from books.api import serializers
+from books import models
 
 
 class AuthorViewSet(viewsets.ModelViewSet):
-    queryset = Author.objects.all().order_by("-created")
-    serializer_class = AuthorSerializer
+    queryset = models.Author.objects.all().order_by("-created")
+    serializer_class = serializers.AuthorSerializer
     permission_classes = [permissions.IsAuthenticated]
 
 
 class BookViewSet(viewsets.ModelViewSet):
-    queryset = Book.objects.all().order_by("-created")
-    serializer_class = BookSerializer
+    queryset = models.Book.objects.all().order_by("-created")
+    serializer_class = serializers.BookSerializer
     permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/books/constants.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python3
+
+BOOKS_TITLES_TO_IGNORE = [
+    "KOReader Quickstart Guide",
+    "zb2rhkSwygt9vjkAEBj7tP5KVgFqejJqsJ2W3bYsrgiiKK8XL",
+    "zb2rhchGpo7P27mofV9hYjT63d9ZaQnbQ6LSfzmkvsYzvARif",
+]
+
+READCOMICSONLINE_URL = "https://readcomicsonline.ru"

vrobbler/apps/books/koreader.py

@@ -1,17 +1,17 @@
-from collections import OrderedDict
 import logging
 import re
 import sqlite3
 from datetime import datetime, timedelta
 from enum import Enum
+from zoneinfo import ZoneInfo
 
-import pytz
 import requests
-from books.models import Author, Book
-from books.openlibrary import get_author_openlibrary_id
+from books.constants import BOOKS_TITLES_TO_IGNORE
 from django.apps import apps
 from django.contrib.auth import get_user_model
+from scrobbles.notifications import ScrobbleNtfyNotification
 from stream_sqlite import stream_sqlite
+from webdav.client import get_webdav_client
 
 logger = logging.getLogger(__name__)
 User = get_user_model()
@@ -62,6 +62,8 @@ def lookup_or_create_authors_from_author_str(ko_author_str: str) -> list:
     """Takes a string of authors from KoReader and returns a list
     of Authors from our database
     """
+    from books.models import Author
+
     author_str_list = ko_author_str.split(", ")
     author_list = []
     for author_str in author_str_list:
@@ -73,36 +75,49 @@ def lookup_or_create_authors_from_author_str(ko_author_str: str) -> list:
 
         author = Author.objects.filter(name=author_str).first()
         if not author:
-            author = Author.objects.create(
-                name=author_str,
-                openlibrary_id=get_author_openlibrary_id(author_str),
-            )
-            author.fix_metadata()
+            author = Author.objects.create(name=author_str)
+            # TODO Move these to async processes after importing
+            # author.fix_metadata()
             logger.debug(f"Created author {author}")
         author_list.append(author)
     return author_list
 
 
 def create_book_from_row(row: list):
+    from books.models import Book
+
     # No KoReader book yet, create it
-    author_str = get_author_str_from_row(row)
+    author_str = get_author_str_from_row(row).replace("\x00", "")
     total_pages = row[KoReaderBookColumn.PAGES.value]
     run_time = total_pages * Book.AVG_PAGE_READING_SECONDS
+    book_title = row[KoReaderBookColumn.TITLE.value].replace("\x00", "")
+    if " - " in book_title:
+        split_title = book_title.split(" - ")
+        book_title = split_title[0]
+        if (not author_str or author_str == "N/A") and len(split_title) > 1:
+            author_str = split_title[1].split("_")[0]
+
+    clean_row = []
+    for value in row:
+        if isinstance(value, str):
+            value = value.replace("\x00", "")
+        clean_row.append(value)
 
     book = Book.objects.create(
-        title=row[KoReaderBookColumn.TITLE.value],
+        title=book_title.replace("_", ":"),
         pages=total_pages,
         koreader_data_by_hash={
             str(row[KoReaderBookColumn.MD5.value]): {
-                "title": row[KoReaderBookColumn.TITLE.value],
+                "title": book_title,
                 "author_str": author_str,
                 "book_id": row[KoReaderBookColumn.ID.value],
-                "pages": total_pages,
+                "raw_row_data": clean_row,
             }
         },
-        run_time_seconds=run_time,
+        base_run_time_seconds=run_time,
     )
-    book.fix_metadata()
+    # TODO Move these to async processes after importing
+    # book.fix_metadata()
 
     # Add authors
     author_list = lookup_or_create_authors_from_author_str(author_str)
@@ -119,15 +134,15 @@ def build_book_map(rows) -> dict:
     primary key IDs for page creation.
 
     """
+    from books.models import Book
+
     book_id_map = {}
 
     for book_row in rows:
-        if (
-            book_row[KoReaderBookColumn.TITLE.value]
-            == "KOReader Quickstart Guide"
-        ):
+        if book_row[KoReaderBookColumn.TITLE.value] in BOOKS_TITLES_TO_IGNORE:
             logger.info(
-                "Ignoring the KOReader quickstart guide. No on wants that."
+                "[build_book_map] Ignoring book title that is likely garbage",
+                extra={"book_row": book_row, "media_type": "Book"},
             )
             continue
         book = Book.objects.filter(
@@ -136,6 +151,15 @@ def build_book_map(rows) -> dict:
             ]
         ).first()
 
+        if not book:
+            title = (
+                book_row[KoReaderBookColumn.TITLE.value]
+                .split(" - ")[0]
+                .lower()
+                .replace("\x00", "")
+            )
+            book = Book.objects.filter(title=title).first()
+
         if not book:
             book = create_book_from_row(book_row)
 
@@ -253,35 +277,19 @@ def build_scrobbles_from_book_map(
                     )
                     continue
 
-                timezone = user.profile.timezone
+                timestamp = user.profile.get_timestamp_with_tz(
+                    datetime.fromtimestamp(int(first_page.get("start_ts")))
+                )
+                stop_timestamp = user.profile.get_timestamp_with_tz(
+                    datetime.fromtimestamp(int(last_page.get("end_ts")))
+                )
 
-                timestamp = datetime.fromtimestamp(
-                    int(first_page.get("start_ts"))
-                ).replace(tzinfo=pytz.timezone(timezone))
-
-                # Add a shim here temporarily to fix imports while we were in France
-                # if date is between 10/15 and 12/15, cast it to Europe/Central
-                if (
-                    datetime(2023, 10, 15).replace(
-                        tzinfo=pytz.timezone("Europe/Paris")
-                    )
-                    <= timestamp
-                    <= datetime(2023, 12, 15).replace(
-                        tzinfo=pytz.timezone("Europe/Paris")
-                    )
-                ):
-                    timezone = "Europe/Paris"
-
-                stop_timestamp = datetime.fromtimestamp(
-                    int(last_page.get("end_ts"))
-                ).replace(tzinfo=pytz.timezone(timezone))
-
-                if (
-                    timestamp.tzinfo._dst.seconds == 0
-                    or stop_timestamp.tzinfo._dst.seconds == 0
-                ):
-                    timestamp = timestamp - timedelta(hours=1)
-                    stop_timestamp = stop_timestamp - timedelta(hours=1)
+                # Adjust for Daylight Saving Time
+                # if timestamp.dst() == timedelta(
+                #    0
+                # ) or stop_timestamp.dst() == timedelta(0):
+                #    timestamp = timestamp - timedelta(hours=1)
+                #    stop_timestamp = stop_timestamp - timedelta(hours=1)
 
                 scrobble = Scrobble.objects.filter(
                     timestamp=timestamp,
@@ -296,8 +304,12 @@ def build_scrobbles_from_book_map(
                     log_data = {
                         "koreader_hash": book_dict.get("hash"),
                         "page_data": scrobble_page_data,
-                        "pages_read": cur_page_number,
+                        "pages_read": len(scrobble_page_data.keys()),
                     }
+                    if hasattr(timestamp.tzinfo, "tzname"):
+                        tz = timestamp.tzinfo.tzname
+                    if hasattr(timestamp.tzinfo, "name"):
+                        tz = timestamp.tzinfo.name
                     scrobbles_to_create.append(
                         Scrobble(
                             book_id=book_id,
@@ -311,13 +323,13 @@ def build_scrobbles_from_book_map(
                             in_progress=False,
                             played_to_completion=True,
                             long_play_complete=False,
-                            timezone=timezone,
+                            timezone=tz,
                         )
                     )
-                    # Then start over
-                    should_create_scrobble = False
-                    playback_position_seconds = 0
-                    scrobble_page_data = {}
+                # Then start over
+                should_create_scrobble = False
+                playback_position_seconds = 0
+                scrobble_page_data = {}
 
             # We accumulate pages for the scrobble until we should create a new one
             scrobble_page_data[cur_page_number] = stats
@@ -353,9 +365,9 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
 
     new_scrobbles = []
     user = User.objects.filter(id=user_id).first()
-    tz = pytz.utc
+    tz = ZoneInfo("UTC")
     if user:
-        tz = user.profile.timezone
+        tz = user.profile.tzinfo
 
     is_os_file = "https://" not in file_path
     if is_os_file:
@@ -397,9 +409,27 @@ def process_koreader_sqlite_file(file_path, user_id) -> list:
     created = []
     if new_scrobbles:
         created = Scrobble.objects.bulk_create(new_scrobbles)
+        if created:
+            ScrobbleNtfyNotification(created[-1]).send()
         fix_long_play_stats_for_scrobbles(created)
         logger.info(
             f"Created {len(created)} scrobbles",
             extra={"created_scrobbles": created},
         )
     return created
+
+
+def fetch_file_from_webdav(user_id: int) -> str:
+    file_path = f"/tmp/{user_id}-koreader-import.sqlite3"
+    client = get_webdav_client(user_id)
+
+    if not client:
+        logger.warning("could not get webdav client for user")
+        # TODO maybe we raise an exception here?
+        return ""
+
+    client.download_sync(
+        remote_path="var/koreader/statistics.sqlite3",
+        local_path=file_path,
+    )
+    return file_path

vrobbler/apps/books/locg.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-from enum import Enum
 from typing import Optional
 from bs4 import BeautifulSoup
 import requests

vrobbler/apps/books/management/commands/cleanup_book_scrobble_times.py

@@ -14,14 +14,13 @@ logger = logging.getLogger(__name__)
 # Grace period between page reads for it to be a new scrobble
 SESSION_GAP_SECONDS = 1800  #  a half hour
 
+
 def update_scrobble_from_page_data(scrobble, commit=True):
     page_list = list(scrobble.book_page_data.items())
     first_page_start_ts = datetime.fromtimestamp(page_list[0][1]["start_ts"])
     last_page_end_ts = datetime.fromtimestamp(page_list[-1][1]["end_ts"])
 
-    if (
-        datetime(2023, 10, 15) <= first_page_start_ts <= datetime(2023, 12, 15)
-    ):
+    if datetime(2023, 10, 15) <= first_page_start_ts <= datetime(2023, 12, 15):
         first_page_start_ts.replace(tzinfo=pytz.timezone("Europe/Paris"))
         last_page_end_ts.replace(tzinfo=pytz.timezone("Europe/Paris"))
     else:
@@ -38,5 +37,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         scrobbles_to_create = []
 
-        for scrobble in Scrobble.objects.filter(media_type="Book", source="KOReader"):
+        for scrobble in Scrobble.objects.filter(
+            media_type="Book", source="KOReader"
+        ):
             update_scrobble_from_page_data(scrobble)

vrobbler/apps/books/metadata.py

@@ -0,0 +1,38 @@
+from typing import Optional
+
+
+YOUTUBE_VIDEO_URL = "https://www.youtube.com/watch?v="
+IMDB_VIDEO_URL = "https://www.imdb.com/title/"
+
+
+class BookType:
+    ...
+
+
+class BookMetadata:
+    title: str
+    run_time_seconds: Optional[int]
+    authors = Optional[str]
+    goodreads_id = Optional[str]
+    koreader_data_by_hash = Optional[dict]
+    isbn = Optional[str]
+    # isbn_13 = Optional[str]
+    # isbn_10 = Optional[str]
+    pages = Optional[int]
+    language = Optional[str]
+    first_publish_year = Optional[int]
+    summary = Optional[str]
+
+    # General
+    cover_url: Optional[str]
+    genres: list[str]
+
+    def __init__(self, title: Optional[str] = ""):
+        self.title = title
+
+    def as_dict_with_authors_cover_and_genres(self) -> tuple:
+        book_dict = vars(self)
+        authors = book_dict.pop("authors")
+        cover = book_dict.pop("cover_url")
+        genres = book_dict.pop("genres")
+        return book_dict, authors, cover, genres

vrobbler/apps/books/migrations/0021_remove_book_koreader_authors_remove_book_koreader_id_and_more.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.16 on 2024-10-17 20:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0020_author_comicvine_data_book_comicvine_data_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="book",
+            name="koreader_authors",
+        ),
+        migrations.RemoveField(
+            model_name="book",
+            name="koreader_id",
+        ),
+        migrations.RemoveField(
+            model_name="book",
+            name="koreader_md5",
+        ),
+    ]

vrobbler/apps/books/migrations/0022_alter_book_run_time_seconds.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.16 on 2025-01-22 03:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "books",
+            "0021_remove_book_koreader_authors_remove_book_koreader_id_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="run_time_seconds",
+            field=models.IntegerField(default=900),
+        ),
+    ]

vrobbler/apps/books/migrations/0023_rename_isbn_book_isbn_13_remove_book_comicvine_data_and_more.py

@@ -0,0 +1,40 @@
+# Generated by Django 4.2.18 on 2025-01-27 04:20
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0022_alter_book_run_time_seconds"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="book",
+            old_name="isbn",
+            new_name="isbn_13",
+        ),
+        migrations.RemoveField(
+            model_name="book",
+            name="comicvine_data",
+        ),
+        migrations.RemoveField(
+            model_name="book",
+            name="goodreads_id",
+        ),
+        migrations.RemoveField(
+            model_name="book",
+            name="locg_slug",
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="isbn_10",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name="book",
+            name="publish_date",
+            field=models.DateField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0024_book_publisher.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.18 on 2025-01-27 04:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        (
+            "books",
+            "0023_rename_isbn_book_isbn_13_remove_book_comicvine_data_and_more",
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="book",
+            name="publisher",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0025_remove_author_amazon_id_and_more.py

@@ -0,0 +1,136 @@
+# Generated by Django 4.2.19 on 2025-02-18 05:11
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import taggit.managers
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("scrobbles", "0067_scrobble_food_alter_scrobble_media_type"),
+        ("books", "0024_book_publisher"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="author",
+            name="amazon_id",
+        ),
+        migrations.RemoveField(
+            model_name="author",
+            name="librarything_id",
+        ),
+        migrations.RemoveField(
+            model_name="author",
+            name="locg_slug",
+        ),
+        migrations.AddField(
+            model_name="author",
+            name="semantic_id",
+            field=models.CharField(blank=True, max_length=50, null=True),
+        ),
+        migrations.CreateModel(
+            name="Paper",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                (
+                    "created",
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name="created"
+                    ),
+                ),
+                (
+                    "modified",
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name="modified"
+                    ),
+                ),
+                (
+                    "uuid",
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                ("run_time_seconds", models.IntegerField(default=900)),
+                (
+                    "run_time_ticks",
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                ("title", models.CharField(max_length=255)),
+                ("semantic_title", models.CharField(max_length=255)),
+                (
+                    "koreader_data_by_hash",
+                    models.JSONField(blank=True, null=True),
+                ),
+                (
+                    "semantic_id",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                (
+                    "arxiv_id",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                (
+                    "corpus_id",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                (
+                    "doi_id",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                ("pages", models.IntegerField(blank=True, null=True)),
+                (
+                    "language",
+                    models.CharField(blank=True, max_length=4, null=True),
+                ),
+                (
+                    "first_publish_year",
+                    models.IntegerField(blank=True, null=True),
+                ),
+                ("publish_date", models.DateField(blank=True, null=True)),
+                (
+                    "journal",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "journal_volume",
+                    models.CharField(blank=True, max_length=50, null=True),
+                ),
+                ("abstract", models.TextField(blank=True, null=True)),
+                ("num_citations", models.IntegerField(blank=True, null=True)),
+                (
+                    "openaccess_pdf_url",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "authors",
+                    models.ManyToManyField(blank=True, to="books.author"),
+                ),
+                (
+                    "genre",
+                    taggit.managers.TaggableManager(
+                        help_text="A comma-separated list of tags.",
+                        through="scrobbles.ObjectWithGenres",
+                        to="scrobbles.Genre",
+                        verbose_name="Tags",
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+        ),
+    ]

vrobbler/apps/books/migrations/0026_alter_paper_semantic_title.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-02-18 05:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0025_remove_author_amazon_id_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="paper",
+            name="semantic_title",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0027_remove_paper_num_citations_paper_tldr.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2.19 on 2025-02-18 05:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0026_alter_paper_semantic_title"),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name="paper",
+            name="num_citations",
+        ),
+        migrations.AddField(
+            model_name="paper",
+            name="tldr",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0028_delete_page.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.2.19 on 2025-04-07 17:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0027_remove_paper_num_citations_paper_tldr"),
+    ]
+
+    operations = [
+        migrations.DeleteModel(
+            name="Page",
+        ),
+    ]

vrobbler/apps/books/migrations/0029_book_comicvine_id_book_issue_number_and_more.py

@@ -0,0 +1,33 @@
+# Generated by Django 4.2.19 on 2025-10-20 18:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('books', '0028_delete_page'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='book',
+            name='comicvine_id',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='issue_number',
+            field=models.IntegerField(blank=True, max_length=5, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='original_title',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='volume_number',
+            field=models.IntegerField(blank=True, max_length=5, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0030_book_readcomics_url.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-10-22 16:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('books', '0029_book_comicvine_id_book_issue_number_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='book',
+            name='readcomics_url',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0031_book_next_readcomics_url.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.19 on 2025-10-22 17:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('books', '0030_book_readcomics_url'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='book',
+            name='next_readcomics_url',
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0032_remove_book_run_time_seconds_and_more.py

@@ -0,0 +1,39 @@
+# Generated by Django 4.2.19 on 2025-10-30 01:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('books', '0031_book_next_readcomics_url'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='book',
+            name='run_time_seconds',
+        ),
+        migrations.RemoveField(
+            model_name='book',
+            name='run_time_ticks',
+        ),
+        migrations.RemoveField(
+            model_name='paper',
+            name='run_time_seconds',
+        ),
+        migrations.RemoveField(
+            model_name='paper',
+            name='run_time_ticks',
+        ),
+        migrations.AddField(
+            model_name='book',
+            name='base_run_time_seconds',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name='paper',
+            name='base_run_time_seconds',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/migrations/0033_alter_book_issue_number_alter_book_volume_number.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.29 on 2026-03-08 05:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("books", "0032_remove_book_run_time_seconds_and_more"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="book",
+            name="issue_number",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name="book",
+            name="volume_number",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/books/models.py

@@ -1,13 +1,19 @@
-from collections import OrderedDict
 import logging
-from datetime import timedelta, datetime
+from collections import OrderedDict
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
 from uuid import uuid4
 
 import requests
+from books.constants import READCOMICSONLINE_URL
 from books.openlibrary import (
     lookup_author_from_openlibrary,
     lookup_book_from_openlibrary,
 )
+from books.sources.google import lookup_book_from_google
+from books.sources.semantic import lookup_paper_from_semantic
+from books.utils import get_comic_issue_url
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.core.files.base import ContentFile
@@ -16,25 +22,25 @@ from django.urls import reverse
 from django_extensions.db.models import TimeStampedModel
 from imagekit.models import ImageSpecField
 from imagekit.processors import ResizeToFit
+from scrobbles.dataclasses import BaseLogData, LongPlayLogData
 from scrobbles.mixins import (
     LongPlayScrobblableMixin,
     ObjectWithGenres,
-    ScrobblableMixin,
+    ScrobblableConstants,
 )
-from scrobbles.utils import get_scrobbles_for_media
+from scrobbles.utils import get_scrobbles_for_media, next_url_if_exists
 from taggit.managers import TaggableManager
 from thefuzz import fuzz
-from vrobbler.apps.books.comicvine import (
-    ComicVineClient,
-    lookup_comic_from_comicvine,
-)
 
 from vrobbler.apps.books.locg import (
     lookup_comic_by_locg_slug,
     lookup_comic_from_locg,
     lookup_comic_writer_by_locg_slug,
 )
-from vrobbler.apps.scrobbles.dataclasses import BookLogData
+from vrobbler.apps.books.sources.comicvine import (
+    ComicVineClient,
+    lookup_comic_from_comicvine,
+)
 
 COMICVINE_API_KEY = getattr(settings, "COMICVINE_API_KEY", "")
 
@@ -43,6 +49,34 @@ User = get_user_model()
 BNULL = {"blank": True, "null": True}
 
 
+@dataclass
+class BookPageLogData(BaseLogData):
+    page_number: Optional[int] = None
+    end_ts: Optional[int] = None
+    start_ts: Optional[int] = None
+    duration: Optional[int] = None
+
+
+@dataclass
+class BookLogData(BaseLogData, LongPlayLogData):
+    koreader_hash: Optional[str] = None
+    page_data: Optional[dict[int, BookPageLogData]] = None
+    pages_read: Optional[int] = None
+    page_start: Optional[int] = None
+    page_end: Optional[int] = None
+    resume_url: Optional[str] = None
+
+    _excluded_fields = {"koreader_hash", "page_data"}
+
+    def avg_seconds_per_page(self):
+        if self.page_data:
+            total_duration = 0
+            for page_num, stats in self.page_data.items():
+                total_duration += stats.get("duration", 0)
+            if total_duration:
+                return int(total_duration / len(self.page_data))
+
+
 class Author(TimeStampedModel):
     name = models.CharField(max_length=255)
     uuid = models.UUIDField(default=uuid4, editable=False, **BNULL)
@@ -62,22 +96,27 @@ class Author(TimeStampedModel):
     )
     bio = models.TextField(**BNULL)
     wikipedia_url = models.CharField(max_length=255, **BNULL)
-    isni = models.CharField(max_length=255, **BNULL)
-    locg_slug = models.CharField(max_length=255, **BNULL)
     wikidata_id = models.CharField(max_length=255, **BNULL)
+    isni = models.CharField(max_length=255, **BNULL)
     goodreads_id = models.CharField(max_length=255, **BNULL)
-    librarything_id = models.CharField(max_length=255, **BNULL)
     comicvine_data = models.JSONField(**BNULL)
-    amazon_id = models.CharField(max_length=255, **BNULL)
+
+    semantic_id = models.CharField(max_length=50, **BNULL)
 
     def __str__(self):
         return f"{self.name}"
 
-    def fix_metadata(self, data_dict: dict = {}):
-        if not data_dict and self.openlibrary_id:
-            data_dict = lookup_author_from_openlibrary(self.openlibrary_id)
+    def enrich_from_semantic(self, overwrite=False):
+        ...
+
+    def enrich_from_google_books(self, overwrite=False):
+        ...
+
+    def enrich_from_openlibrary(self, overwrite=False):
+        data_dict = lookup_author_from_openlibrary(self.openlibrary_id)
 
         if not data_dict or not data_dict.get("name"):
+            logger.warning("Could not find author on openlibrary")
             return
 
         headshot_url = data_dict.pop("author_headshot_url", "")
@@ -99,21 +138,25 @@ class Book(LongPlayScrobblableMixin):
     )
 
     title = models.CharField(max_length=255)
+    original_title = models.CharField(max_length=255, **BNULL)
     authors = models.ManyToManyField(Author, blank=True)
-    goodreads_id = models.CharField(max_length=255, **BNULL)
-    # All individual koreader fields are deprecated
-    koreader_id = models.IntegerField(**BNULL)
-    koreader_authors = models.CharField(max_length=255, **BNULL)
-    koreader_md5 = models.CharField(max_length=255, **BNULL)
     koreader_data_by_hash = models.JSONField(**BNULL)
-    isbn = models.CharField(max_length=255, **BNULL)
+    isbn_13 = models.CharField(max_length=255, **BNULL)
+    isbn_10 = models.CharField(max_length=255, **BNULL)
     pages = models.IntegerField(**BNULL)
     language = models.CharField(max_length=4, **BNULL)
     first_publish_year = models.IntegerField(**BNULL)
+    publish_date = models.DateField(**BNULL)
+    publisher = models.CharField(max_length=255, **BNULL)
     first_sentence = models.TextField(**BNULL)
+    # ComicVine
+    comicvine_id = models.CharField(max_length=255, **BNULL)
+    readcomics_url = models.CharField(max_length=255, **BNULL)
+    next_readcomics_url = models.CharField(max_length=255, **BNULL)
+    issue_number = models.IntegerField(**BNULL)
+    volume_number = models.IntegerField(**BNULL)
+    # OpenLibrary
     openlibrary_id = models.CharField(max_length=255, **BNULL)
-    locg_slug = models.CharField(max_length=255, **BNULL)
-    comicvine_data = models.JSONField(**BNULL)
     cover = models.ImageField(upload_to="books/covers/", **BNULL)
     cover_small = ImageSpecField(
         source="cover",
@@ -131,13 +174,21 @@ class Book(LongPlayScrobblableMixin):
 
     genre = TaggableManager(through=ObjectWithGenres)
 
-    def __str__(self):
+    def __str__(self) -> str:
+        if self.issue_number and "Issue" not in str(self.title):
+            return f"{self.title} - Issue {self.issue_number}"
+        if self.volume_number and "Volume" not in str(self.title):
+            return f"{self.title} - Volume {self.volume_number}"
         return f"{self.title}"
 
     @property
     def subtitle(self):
         return f" by {self.author}"
 
+    @property
+    def strings(self) -> ScrobblableConstants:
+        return ScrobblableConstants(verb="Reading", tags="book")
+
     @property
     def logdata_cls(self):
         return BookLogData
@@ -149,12 +200,129 @@ class Book(LongPlayScrobblableMixin):
             url = self.cover_medium.url
         return url
 
-    def get_start_url(self):
-        return reverse("scrobbles:start", kwargs={"uuid": self.uuid})
-
     def get_absolute_url(self):
         return reverse("books:book_detail", kwargs={"slug": self.uuid})
 
+    @classmethod
+    def get_from_comicvine(
+        cls, title: str, overwrite: bool = False, force_new: bool = False
+    ) -> "Book":
+        book, created = cls.objects.get_or_create(title=title)
+
+        if not created:
+            return book
+
+        book_dict = lookup_comic_from_comicvine(title)
+
+        if created or overwrite:
+            author_list = []
+            author_dicts = book_dict.pop("author_dicts")
+            if author_dicts:
+                for author_dict in author_dicts:
+                    if author_dict.get("authorId"):
+                        author, a_created = Author.objects.get_or_create(
+                            semantic_id=author_dict.get("authorId")
+                        )
+                        author_list.append(author)
+                        if a_created:
+                            author.name = author_dict.get("name")
+                            author.save()
+                            # TODO enrich author?
+                            ...
+
+            for k, v in book_dict.items():
+                setattr(book, k, v)
+            book.save()
+
+            if author_list:
+                book.authors.add(*author_list)
+            genres = book_dict.pop("genres", [])
+            if genres:
+                book.genre.add(*genres)
+        return book
+
+    @classmethod
+    def find_or_create(
+        cls,
+        title: str,
+        url: str = "",
+        enrich: bool = False,
+        commit: bool = True,
+    ):
+        """Given a title, get a Book instance.
+
+        If the book is not already in our database, or overwrite is True,
+        this method will enrich the Book with data from Google.
+
+        By default this method will also save the data back to the model. If you'd
+        like to batch create, use commit=False and you'll get an unsaved but enriched
+        instance back which you can then save at your convenience."""
+        # TODO use either a Google Books id identifier or author name like for tracks
+        book, created = cls.objects.get_or_create(original_title=title)
+        if not created:
+            logger.info(
+                "Found exact match for book by title", extra={"title": title}
+            )
+
+        if not enrich:
+            logger.info(
+                "Found book by title, but not enriching",
+                extra={"title": title},
+            )
+            return book
+
+        book_dict = None
+        if READCOMICSONLINE_URL in url:
+            book_dict = lookup_comic_from_comicvine(title)
+            book_dict["readcomics_url"] = get_comic_issue_url(url)
+            book_dict["next_readcomics_url"] = next_url_if_exists(
+                book_dict["readcomics_url"]
+            )
+
+        if not book_dict:
+            book_dict = lookup_book_from_google(title)
+
+        if not book_dict:
+            logger.warning(
+                "No book found in any source, using data as is",
+                extra={"title": title},
+            )
+
+        author_list = []
+        authors = book_dict.pop("authors", [])
+        cover_url = book_dict.pop("cover_url", "")
+        genres = book_dict.pop("generes", [])
+
+        if authors:
+            for author_str in authors:
+                if author_str:
+                    author, a_created = Author.objects.get_or_create(
+                        name=author_str
+                    )
+                    author_list.append(author)
+                    if a_created:
+                        # TODO enrich author
+                        ...
+
+        for k, v in book_dict.items():
+            setattr(book, k, v)
+
+        if commit:
+            book.save()
+
+            book.save_image_from_url(cover_url)
+            book.genre.add(*genres)
+            book.authors.add(*author_list)
+
+        return book
+
+    def save_image_from_url(self, url: str, force_update: bool = False):
+        if url and (not self.cover or force_update):
+            r = requests.get(url)
+            if r.status_code == 200:
+                fname = f"{self.title}_{self.uuid}.jpg"
+                self.cover.save(fname, ContentFile(r.content), save=True)
+
     def fix_metadata(self, data: dict = {}, force_update=False):
         if (not self.openlibrary_id or not self.locg_slug) or force_update:
             author_name = ""
@@ -245,7 +413,7 @@ class Book(LongPlayScrobblableMixin):
                     self.cover.save(fname, ContentFile(r.content), save=True)
 
             if self.pages:
-                self.run_time_seconds = int(self.pages) * int(
+                self.base_run_time_seconds = int(self.pages) * int(
                     self.AVG_PAGE_READING_SECONDS
                 )
 
@@ -329,112 +497,67 @@ class Book(LongPlayScrobblableMixin):
             progress = int((last_scrobble.last_page_read / self.pages) * 100)
         return progress
 
+
+class Paper(LongPlayScrobblableMixin):
+    """Keeps track of Academic Papers"""
+
+    COMPLETION_PERCENT = getattr(settings, "PAPER_COMPLETION_PERCENT", 60)
+    AVG_PAGE_READING_SECONDS = getattr(
+        settings, "AVERAGE_PAGE_READING_SECONDS", 60
+    )
+
+    title = models.CharField(max_length=255)
+    semantic_title = models.CharField(max_length=255, **BNULL)
+    authors = models.ManyToManyField(Author, blank=True)
+    koreader_data_by_hash = models.JSONField(**BNULL)
+    arxiv_id = models.CharField(max_length=50, **BNULL)
+    semantic_id = models.CharField(max_length=50, **BNULL)
+    arxiv_id = models.CharField(max_length=50, **BNULL)
+    corpus_id = models.CharField(max_length=50, **BNULL)
+    doi_id = models.CharField(max_length=50, **BNULL)
+    pages = models.IntegerField(**BNULL)
+    language = models.CharField(max_length=4, **BNULL)
+    first_publish_year = models.IntegerField(**BNULL)
+    publish_date = models.DateField(**BNULL)
+    journal = models.CharField(max_length=255, **BNULL)
+    journal_volume = models.CharField(max_length=50, **BNULL)
+    abstract = models.TextField(**BNULL)
+    tldr = models.CharField(max_length=255, **BNULL)
+    openaccess_pdf_url = models.CharField(max_length=255, **BNULL)
+
+    genre = TaggableManager(through=ObjectWithGenres)
+
     @classmethod
-    def find_or_create(cls, lookup_id: str, author: str = "") -> "Book":
-        book = cls.objects.filter(openlibrary_id=lookup_id).first()
+    def get_from_semantic(cls, title: str, overwrite: bool = False) -> "Paper":
+        paper, created = cls.objects.get_or_create(title=title)
+        if not created and not overwrite:
+            return paper
 
-        if not book:
-            data = lookup_book_from_openlibrary(lookup_id, author)
+        paper_dict = lookup_paper_from_semantic(title)
 
-            if not data:
-                logger.error(
-                    f"No book found on openlibrary, or in our database for {lookup_id}"
-                )
-                return book
+        if created or overwrite:
+            author_list = []
+            author_dicts = paper_dict.pop("author_dicts")
+            if author_dicts:
+                for author_dict in author_dicts:
+                    if author_dict.get("authorId"):
+                        author, a_created = Author.objects.get_or_create(
+                            semantic_id=author_dict.get("authorId")
+                        )
+                        author_list.append(author)
+                        if a_created:
+                            author.name = author_dict.get("name")
+                            author.save()
+                            # TODO enrich author?
+                            ...
 
-            book, book_created = cls.objects.get_or_create(isbn=data["isbn"])
-            if book_created:
-                book.fix_metadata(data=data)
+            for k, v in paper_dict.items():
+                setattr(paper, k, v)
+            paper.save()
 
-        return book
-
-
-class Page(TimeStampedModel):
-    """DEPRECATED, we need to migrate pages into page_data on scrobbles and move on"""
-
-    book = models.ForeignKey(Book, on_delete=models.CASCADE)
-    number = models.IntegerField()
-    user = models.ForeignKey(User, on_delete=models.CASCADE)
-    start_time = models.DateTimeField(**BNULL)
-    end_time = models.DateTimeField(**BNULL)
-    duration_seconds = models.IntegerField(**BNULL)
-
-    class Meta:
-        unique_together = (
-            "book",
-            "number",
-        )
-
-    def __str__(self):
-        return f"Page {self.number} of {self.book.pages} in {self.book.title}"
-
-    def save(self, *args, **kwargs):
-        if not self.end_time and self.duration_seconds:
-            self._set_end_time()
-
-        return super(Page, self).save(*args, **kwargs)
-
-    @property
-    def next(self):
-        page = self.book.page_set.filter(number=self.number + 1).first()
-        if not page:
-            page = (
-                self.book.page_set.filter(created__gt=self.created)
-                .order_by("created")
-                .first()
-            )
-        return page
-
-    @property
-    def previous(self):
-        page = self.book.page_set.filter(number=self.number - 1).first()
-        if not page:
-            page = (
-                self.book.page_set.filter(created__lt=self.created)
-                .order_by("-created")
-                .first()
-            )
-        return page
-
-    @property
-    def seconds_to_next_page(self) -> int:
-        seconds = 999999  # Effectively infnity time as we have no next
-        if not self.end_time:
-            self._set_end_time()
-        if self.next:
-            seconds = (self.next.start_time - self.end_time).seconds
-        return seconds
-
-    @property
-    def is_scrobblable(self) -> bool:
-        """A page defines the start of a scrobble if the seconds to next page
-        are greater than an hour, or 3600 seconds, and it's not a single page,
-        so the next seconds to next_page is less than an hour as well.
-
-        As a special case, the first recorded page is a scrobble, so we establish
-        when the book was started.
-
-        """
-        is_scrobblable = False
-        over_an_hour_since_last_page = False
-        if not self.previous:
-            is_scrobblable = True
-
-        if self.previous:
-            over_an_hour_since_last_page = (
-                self.previous.seconds_to_next_page >= 3600
-            )
-        blip = self.seconds_to_next_page >= 3600
-
-        if over_an_hour_since_last_page and not blip:
-            is_scrobblable = True
-        return is_scrobblable
-
-    def _set_end_time(self) -> None:
-        if self.end_time:
-            return
-
-        self.end_time = self.start_time + timedelta(
-            seconds=self.duration_seconds
-        )
-        self.save(update_fields=["end_time"])
+            if author_list:
+                paper.authors.add(*author_list)
+            genres = paper_dict.pop("genres", [])
+            if genres:
+                paper.genre.add(*genres)
+        return paper

vrobbler/apps/books/sources/comicvine.py

@@ -3,7 +3,6 @@ ComicVine API Information & Documentation:
 https://comicvine.gamespot.com/api/
 https://comicvine.gamespot.com/api/documentation
 """
-import json
 import logging
 from django.conf import settings
 
@@ -200,34 +199,72 @@ class ComicVineClient(object):
 
 
 def lookup_comic_from_comicvine(title: str) -> dict:
+    original_title = title
+
+    issue_number = None
+    volume_nubmer = None
+    resource_type = "issue"
+    if "Issue " in title:
+        resource_type = "issue"
+        issue_number = title.split("Issue ")[1]
+    volume_number = None
+    if "Volume " in title:
+        resource_type = "volume"
+        volume_number = title.split("Volume ")[1]
+
     api_key = getattr(settings, "COMICVINE_API_KEY", "")
     if not api_key:
-        logger.warn("No ComicVine API key configured, not looking anything up")
+        logger.warning(
+            "No ComicVine API key configured, not looking anything up"
+        )
         return {}
 
     client = ComicVineClient(
         api_key=getattr(settings, "COMICVINE_API_KEY", None)
     )
-    result = [
-        r
-        for r in client.search(title).get("results")
-        if r.get("resource_type") == "volume"
-    ][0]
 
-    if "volume" not in result.keys():
-        logger.warn("No result found on ComicVine", extra={"title": title})
+    raw_results = client.search(title).get("results")
+    results = [
+        r for r in raw_results if r.get("resource_type") == resource_type
+    ]
+    if not results:
+        logger.warning("No comic found on ComicVine")
         return {}
 
-    title = " ".join([result.get("volume").get("name"), result.get("name)")])
+    found_result = None
+    for result in results:
+        if result.get("issue_number") == str(issue_number):
+            found_result = result
+            break
+        if result.get("volume_number") == str(volume_number):
+            found_result = result
+            break
+
+    if not found_result:
+        found_result = results[0]
+
+    logger.info("ComicVine results", extra={"results": results})
+
+    if not found_result:
+        logger.warning("No matches found on ComicVine")
+        return {}
+
+    title = found_result.get("name")
+
+    if found_result.get("volume"):
+        title = found_result.get("volume").get("name")
+
     data_dict = {
         "title": title,
-        "cover_url": result.get("image").get("original_url"),
-        "comicvine_data": {
-            "id": result.get("id"),
-            "site_detail_url": result.get("site_detail_url"),
-            "description": result.get("description"),
-            "image": result.get("image").get("original_url"),
-        },
+        "original_title": original_title,
+        "issue_number": found_result.get("issue_number"),
+        "volume_number": found_result.get("volume_number"),
+        "cover_url": found_result.get("image").get("original_url"),
+        "comicvine_id": found_result.get("id"),
+        "comicvine_data": found_result,
+        "summary": found_result.get("description"),
+        "publish_date": found_result.get("cover_date"),
+        "first_publish_year": found_result.get("cover_date", "")[:4],
     }
 
     return data_dict

vrobbler/apps/books/sources/google.py

@@ -0,0 +1,76 @@
+import json
+import logging
+
+import pendulum
+import requests
+from django.conf import settings
+
+API_KEY = settings.GOOGLE_API_KEY
+GOOGLE_BOOKS_URL = (
+    'https://www.googleapis.com/books/v1/volumes?q="{title}"&key={key}'
+)
+
+logger = logging.getLogger(__name__)
+
+
+def lookup_book_from_google(title: str) -> dict:
+    book_dict = {"title": title}
+
+    url = GOOGLE_BOOKS_URL.format(title=title, key=API_KEY)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        logger.warning(
+            "Bad response from Google", extra={"response": response}
+        )
+        return book_dict
+
+    google_result = (
+        json.loads(response.content).get("items", [{}])[0].get("volumeInfo")
+    )
+    if not google_result:
+        return {}
+
+    publish_date = pendulum.parse(google_result.get("publishedDate"))
+
+    isbn_13 = ""
+    isbn_10 = ""
+    for ident in google_result.get("industryIdentifiers", []):
+        if ident.get("type") == "ISBN_13":
+            isbn_13 = ident.get("identifier")
+        if ident.get("type") == "ISBN_10":
+            isbn_10 = ident.get("identifier")
+    # TODO this may lead to issues with the first get if Google changes our title
+    # book_metadata.title = google_result.get("title")
+    # if google_result.get("subtitle"):
+    #    book_metadata["title"] = ": ".join(
+    #        [google_result.get("title"), google_result.get("subtitle")]
+    #    )
+    # book_dict["subtitle"] = google_result.get("subtitle")
+    book_dict["authors"] = google_result.get("authors")
+    book_dict["publisher"] = google_result.get("publisher")
+    book_dict["first_publish_year"] = publish_date.year
+    book_dict["pages"] = google_result.get("pageCount")
+    book_dict["isbn_13"] = isbn_13
+    book_dict["isbn_10"] = isbn_10
+    book_dict["publish_date"] = google_result.get("publishedDate")
+    if len(book_dict["publish_date"]) == 4:
+        book_dict["publish_date"] = f"{book_dict['publish_date']}-1-1"
+    book_dict["language"] = google_result.get("language")
+    book_dict["summary"] = google_result.get("description")
+    book_dict["genres"] = google_result.get("categories")
+    book_dict["cover_url"] = (
+        google_result.get("imageLinks", {})
+        .get("thumbnail", "")
+        .replace("zoom=1", "zoom=15")
+        .replace("&edge=curl", "")
+    )
+
+    book_dict["base_run_time_seconds"] = 3600
+    if book_dict.get("pages"):
+        book_dict["base_run_time_seconds"] = book_dict.get(
+            "pages", 10
+        ) * getattr(settings, "AVERAGE_PAGE_READING_SECONDS", 60)
+
+    return book_dict

vrobbler/apps/books/sources/semantic.py

@@ -0,0 +1,76 @@
+import json
+import logging
+from datetime import datetime
+
+import requests
+from django.conf import settings
+
+PAPER_SEARCH_URL = (
+    "https://api.semanticscholar.org/graph/v1/paper/search/match?query={}"
+)
+PAPER_DETAIL_URL = "https://api.semanticscholar.org/graph/v1/paper/{}?fields=title,authors,url,year,abstract,externalIds,citationCount,referenceCount,journal,fieldsOfStudy,publicationDate,openAccessPdf"
+
+logger = logging.getLogger(__name__)
+
+
+def get_api_result(url):
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    response = requests.get(url, headers=headers)
+
+    if response.status_code != 200:
+        logger.warning(
+            "Bad response from Semantic", extra={"response": response}
+        )
+        return None
+
+    return response
+
+
+def lookup_paper_from_semantic(title: str) -> dict:
+    paper_dict = {"title": title}
+
+    response = get_api_result(PAPER_SEARCH_URL.format(title))
+    if not response:
+        return paper_dict
+
+    semantic_id = json.loads(response.content).get("data")[0].get("paperId")
+    response = get_api_result(PAPER_DETAIL_URL.format(semantic_id))
+
+    result = json.loads(response.content)
+
+    if not result:
+        return paper_dict
+
+    page_str = result.get("journal", {}).get("pages")
+    if page_str:
+        try:
+            start_page = page_str.split(" - ")[0]
+            end_page = page_str.split(" - ")[1]
+            paper_dict["pages"] = int(end_page) - int(start_page)
+        except IndexError:
+            pass
+
+    paper_dict["semantic_id"] = result.get("paperId")
+    paper_dict["doi_id"] = result.get("externalIds", {}).get("DOI")
+    paper_dict["arxiv_id"] = result.get("externalIds", {}).get("ArXiv")
+    paper_dict["pubmed_id"] = result.get("externalIds", {}).get("PubMed")
+    paper_dict["corpus_id"] = result.get("externalIds", {}).get("CorpusId")
+    paper_dict["semantic_title"] = result.get("title")
+    paper_dict["first_publish_year"] = result.get("year")
+    paper_dict["publish_date"] = datetime.strptime(
+        result.get("publicationDate", "1950-01-01"), "%Y-%m-%d"
+    )
+    paper_dict["abstract"] = result.get("abstract")
+    paper_dict["tldr"] = result.get("bib", {}).get("abstract")
+    paper_dict["journal"] = result.get("journal", {}).get("name")
+    paper_dict["journal_volume"] = result.get("journal", {}).get("volume")
+    paper_dict["openaccess_pdf_url"] = result.get("openAccessPdf", {}).get(
+        "url"
+    )
+    paper_dict["base_run_time_seconds"] = paper_dict.get(
+        "pages", 10
+    ) * getattr(settings, "AVERAGE_PAGE_READING_SECONDS", 60)
+    paper_dict["author_dicts"] = result.get("authors")
+    paper_dict["genres"] = result.get("fieldsOfStudy")
+
+    return paper_dict

vrobbler/apps/books/tests/test_koreader.py

@@ -1,11 +1,16 @@
 import pytest
 from unittest import mock
+from datetime import datetime
 
 from books.koreader import (
     KoReaderBookColumn,
     build_book_map,
     build_page_data,
     build_scrobbles_from_book_map,
+    get_author_str_from_row,
+    lookup_or_create_authors_from_author_str,
+    create_book_from_row,
+    process_koreader_sqlite_file,
 )
 
 
@@ -52,3 +57,157 @@ def test_build_scrobbles_from_pages(
     assert len(scrobbles[3].logdata.page_data.keys()) == 20
     assert len(scrobbles[4].logdata.page_data.keys()) == 20
     assert len(scrobbles[5].logdata.page_data.keys()) == 18
+
+
+def test_get_author_str_from_row():
+    row = [
+        1,
+        "Test Book",
+        "John Smith\nJane Doe",
+        0,
+        0,
+        0,
+        300,
+        "",
+        "",
+        "abc123",
+        0,
+        120,
+    ]
+    result = get_author_str_from_row(row)
+    assert result == "John Smith, Jane Doe"
+
+
+def test_get_author_str_from_row_strips_middle_initials():
+    row = [
+        1,
+        "Test Book",
+        "John A. Smith",
+        0,
+        0,
+        0,
+        300,
+        "",
+        "",
+        "abc123",
+        0,
+        120,
+    ]
+    result = get_author_str_from_row(row)
+    assert result == "John Smith"
+
+
+@pytest.mark.django_db
+def test_lookup_or_create_authors_from_author_str_creates_new_author():
+    author = lookup_or_create_authors_from_author_str("New Author")
+    assert len(author) == 1
+    assert author[0].name == "New Author"
+
+
+@pytest.mark.django_db
+def test_lookup_or_create_authors_from_author_str_finds_existing():
+    from books.models import Author
+
+    Author.objects.create(name="Existing Author")
+    author = lookup_or_create_authors_from_author_str("Existing Author")
+    assert len(author) == 1
+    assert author[0].name == "Existing Author"
+
+
+@pytest.mark.django_db
+def test_lookup_or_create_authors_from_author_str_ignores_na():
+    author = lookup_or_create_authors_from_author_str("N/A")
+    assert len(author) == 0
+
+
+@pytest.mark.django_db
+def test_lookup_or_create_authors_from_author_str_handles_multiple():
+    author = lookup_or_create_authors_from_author_str("Author One, Author Two")
+    assert len(author) == 2
+
+
+@pytest.mark.django_db
+def test_create_book_from_row():
+    row = [
+        1,
+        "Test Book - Author Name",
+        "Test Author",
+        0,
+        0,
+        0,
+        300,
+        "",
+        "",
+        "abc123def456",
+        3600,
+        120,
+    ]
+    book = create_book_from_row(row)
+    assert book.title == "Test Book"
+    assert book.pages == 300
+    assert "abc123def456" in book.koreader_data_by_hash
+    assert book.authors.count() == 1
+
+
+@pytest.mark.django_db
+def test_create_book_from_row_handles_title_with_author_suffix():
+    row = [
+        1,
+        "Test Book - Author_Name",
+        "N/A",
+        0,
+        0,
+        0,
+        300,
+        "",
+        "",
+        "abc123def456",
+        3600,
+        120,
+    ]
+    book = create_book_from_row(row)
+    assert book.title == "Test Book"
+    assert book.authors.count() == 1
+    assert book.authors.first().name == "Author"
+
+
+@pytest.mark.django_db
+def test_create_book_from_row_handles_null_bytes():
+    row = [
+        1,
+        "Test\x00Book",
+        "Test\x00Author",
+        0,
+        0,
+        0,
+        300,
+        "",
+        "",
+        "abc123def456",
+        3600,
+        120,
+    ]
+    book = create_book_from_row(row)
+    assert "TestBook" in book.title
+    assert "TestAuthor" in book.authors.first().name
+
+
+@pytest.mark.django_db
+def test_build_book_map_ignores_garbage_titles(koreader_rows):
+    garbage_row = [
+        999,
+        "KOReader Quickstart Guide",
+        "N/A",
+        0,
+        0,
+        0,
+        0,
+        "",
+        "",
+        "garbage123",
+        0,
+        0,
+    ]
+    book_map = build_book_map(koreader_rows.BOOK_ROWS + [garbage_row])
+    assert len(book_map) == 1
+    assert 999 not in book_map

vrobbler/apps/books/tests/test_openlibrary.py

@@ -5,6 +5,7 @@ import pytest
 from books.openlibrary import lookup_book_from_openlibrary
 
 
+@pytest.mark.skip()
 def test_lookup_modern_book():
     book = lookup_book_from_openlibrary("Matrix", "Lauren Groff")
     assert book.get("title") == "Matrix"
@@ -12,6 +13,7 @@ def test_lookup_modern_book():
     assert book.get("ol_author_id") == "OL3675729A"
 
 
+@pytest.mark.skip()
 def test_lookup_classic_book():
     book = lookup_book_from_openlibrary(
         "The Life of Castruccio Castracani", "Machiavelli"
@@ -21,6 +23,7 @@ def test_lookup_classic_book():
     assert book.get("ol_author_id") == "OL23135A"
 
 
+@pytest.mark.skip()
 def test_lookup_foreign_book():
     book = lookup_book_from_openlibrary("Ravagé", "René Barjavel")
     assert book.get("title") == "Ravage"

vrobbler/apps/books/urls.py

@@ -5,14 +5,14 @@ app_name = "books"
 
 
 urlpatterns = [
-    path("book/", views.BookListView.as_view(), name="book_list"),
+    path("books/", views.BookListView.as_view(), name="book_list"),
     path(
-        "book/<slug:slug>/",
+        "books/<slug:slug>/",
         views.BookDetailView.as_view(),
         name="book_detail",
     ),
     path(
-        "author/<slug:slug>/",
+        "authors/<slug:slug>/",
         views.AuthorDetailView.as_view(),
         name="author_detail",
     ),

vrobbler/apps/books/utils.py

@@ -0,0 +1,61 @@
+import re
+from urllib.parse import urlparse, urlunparse
+
+from titlecase import titlecase
+
+
+def parse_readcomicsonline_uri(uri: str) -> tuple:
+    try:
+        path = uri.split("comic/")[1]
+    except IndexError:
+        return "", "", ""
+
+    parts = path.split("/")
+    title = ""
+    volume = 1
+    page = 1
+    if len(parts) == 2:
+        title = titlecase(parts[0].replace("-", " "))
+        volume = parts[1]
+    if len(parts) == 3:
+        title = titlecase(parts[0].replace("-", " "))
+        volume = parts[1]
+        page = parts[2]
+
+    return title, volume, page
+
+
+def get_comic_issue_url(url: str) -> str:
+    parsed = urlparse(url)
+    parts = [p for p in parsed.path.strip("/").split("/") if p]
+
+    # Find the index of "comic"
+    try:
+        comic_index = parts.index("comic")
+    except ValueError:
+        raise ValueError("URL does not contain '/comic/' segment")
+
+    # Extract title (next part after 'comic')
+    if len(parts) <= comic_index + 1:
+        raise ValueError("No comic title found after '/comic/'")
+    title = parts[comic_index + 1]
+
+    # Look for the first numeric segment after the title
+    number = None
+    for segment in parts[comic_index + 2 :]:
+        if segment.isdigit():
+            number = segment
+            break
+
+    # Build normalized path
+    new_parts = ["comic", title]
+    if number:
+        new_parts.append(number)
+
+    normalized_path = "/" + "/".join(new_parts)
+
+    # Rebuild full URL (same scheme and host)
+    simplified_url = urlunparse(
+        parsed._replace(path=normalized_path, query="", fragment="")
+    )
+    return simplified_url

vrobbler/apps/bricksets/api/serializers.py

@@ -0,0 +1,8 @@
+from rest_framework import serializers
+from bricksets.models import BrickSet
+
+
+class BrickSetSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = BrickSet
+        fields = "__all__"

vrobbler/apps/bricksets/api/views.py

@@ -0,0 +1,9 @@
+from rest_framework import permissions, viewsets
+from bricksets.api.serializers import BrickSetSerializer
+from bricksets.models import BrickSet
+
+
+class BrickSetViewSet(viewsets.ModelViewSet):
+    queryset = BrickSet.objects.all().order_by("-created")
+    serializer_class = BrickSetSerializer
+    permission_classes = [permissions.IsAuthenticated]

vrobbler/apps/bricksets/migrations/0002_alter_brickset_run_time_seconds.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2025-01-22 03:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bricksets", "0001_initial"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="brickset",
+            name="run_time_seconds",
+            field=models.IntegerField(default=900),
+        ),
+    ]

vrobbler/apps/bricksets/migrations/0003_remove_brickset_run_time_seconds_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.19 on 2025-10-30 01:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('bricksets', '0002_alter_brickset_run_time_seconds'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='brickset',
+            name='run_time_seconds',
+        ),
+        migrations.RemoveField(
+            model_name='brickset',
+            name='run_time_ticks',
+        ),
+        migrations.AddField(
+            model_name='brickset',
+            name='base_run_time_seconds',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/bricksets/models.py

@@ -1,22 +1,34 @@
-from django.apps import apps
+from dataclasses import dataclass
+
 from django.db import models
 from django.urls import reverse
-from django_extensions.db.models import TimeStampedModel
 from imagekit.models import ImageSpecField
 from imagekit.processors import ResizeToFit
-from scrobbles.dataclasses import BrickSetLogData
 from scrobbles.mixins import LongPlayScrobblableMixin
 
+from vrobbler.apps.scrobbles.dataclasses import (
+    BaseLogData,
+    LongPlayLogData,
+    WithPeopleLogData,
+)
+
 BNULL = {"blank": True, "null": True}
 
 
+@dataclass
+class BrickSetLogData(BaseLogData, WithPeopleLogData, LongPlayLogData):
+    pass
+
+
 class BrickSet(LongPlayScrobblableMixin):
     """"""
 
     number = models.CharField(max_length=10, **BNULL)
     release_year = models.IntegerField(**BNULL)
     piece_count = models.IntegerField(**BNULL)
-    brickset_rating = models.DecimalField(max_digits=3, decimal_places=1, **BNULL)
+    brickset_rating = models.DecimalField(
+        max_digits=3, decimal_places=1, **BNULL
+    )
     lego_item_number = models.CharField(max_length=10, **BNULL)
     box_image = models.ImageField(upload_to="brickset/boxes/", **BNULL)
     box_image_small = ImageSpecField(
@@ -48,13 +60,23 @@ class BrickSet(LongPlayScrobblableMixin):
     def get_absolute_url(self):
         return reverse("bricksets:brickset_detail", kwargs={"slug": self.uuid})
 
+    def __str__(self) -> str:
+        name = str(self.title)
+        if not self.title:
+            name = str(self.number)
+        return name
+
     @property
     def logdata_cls(self):
         return BrickSetLogData
 
     @classmethod
-    def find_or_create(cls, title: str) -> "BrickSet":
-        return cls.objects.filter(title=title).first()
+    def find_or_create(cls, brickset_id: str) -> "BrickSet":
+        brickset = cls.objects.filter(number=brickset_id).first()
+        if not brickset:
+            # TODO: enrich this from the website
+            brickset = cls.objects.create(number=brickset_id)
+        return brickset
 
     @property
     def primary_image_url(self) -> str:

vrobbler/apps/foods/admin.py

@@ -0,0 +1,28 @@
+from foods.models import Food, FoodCategory
+from django.contrib import admin
+from scrobbles.admin import ScrobbleInline
+
+
+class FoodInline(admin.TabularInline):
+    model = Food
+    extra = 0
+
+
+@admin.register(FoodCategory)
+class FoodCategoryAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    search_fields = ("name",)
+
+
+@admin.register(Food)
+class FoodAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "uuid",
+        "title",
+    )
+    ordering = ("-created",)
+    search_fields = ("title",)
+    inlines = [
+        ScrobbleInline,
+    ]

vrobbler/apps/foods/allrecipe.py

@@ -0,0 +1,143 @@
+import json
+import logging
+import urllib
+from typing import Optional
+
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+
+ALLRECIPE_URL = "https://allrecipe.com/{id}"
+
+
+def get_first(key: str, result: dict) -> str:
+    obj = ""
+    if obj_list := result.get(key):
+        obj = obj_list[0]
+    return obj
+
+
+def get_title_from_soup(soup) -> str:
+    title = ""
+    try:
+        title = soup.find("h1").get_text()
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return title
+
+
+def get_description_from_soup(soup) -> str:
+    desc = ""
+    try:
+        desc = (
+            soup.find(class_="beer-descrption-read-less")
+            .get_text()
+            .replace("Show Less", "")
+            .strip()
+        )
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return desc
+
+
+def get_styles_from_soup(soup) -> list[str]:
+    styles = []
+    try:
+        styles = soup.find("p", class_="style").get_text().split(" - ")
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return styles
+
+
+def get_abv_from_soup(soup) -> Optional[float]:
+    abv = None
+    try:
+        abv = soup.find(class_="abv").get_text()
+        if abv:
+            abv = float(abv.strip("\n").strip("% ABV").strip())
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    except TypeError:
+        pass
+    return abv
+
+
+def get_ibu_from_soup(soup) -> Optional[int]:
+    ibu = None
+    try:
+        ibu = soup.find(class_="ibu").get_text()
+        if ibu:
+            ibu = int(ibu.strip("\n").strip(" IBU").strip())
+    except AttributeError:
+        pass
+    except ValueError:
+        ibu = None
+    return ibu
+
+
+def get_rating_from_soup(soup) -> str:
+    rating = ""
+    try:
+        rating = float(
+            soup.find(class_="num").get_text().strip("(").strip(")")
+        )
+    except AttributeError:
+        rating = None
+    except ValueError:
+        rating = None
+    return rating
+
+
+def get_producer_id_from_soup(soup) -> str:
+    id = ""
+    try:
+        id = soup.find(class_="brewery").find("a")["href"].strip("/")
+    except ValueError:
+        pass
+    except IndexError:
+        pass
+    return id
+
+
+def get_producer_name_from_soup(soup) -> str:
+    name = ""
+    try:
+        name = soup.find(class_="brewery").find("a").get_text()
+    except AttributeError:
+        pass
+    except ValueError:
+        pass
+    return name
+
+
+def get_food_from_allrecipe_id(allrecipe_id: str) -> dict:
+    url = ALLRECIPE_URL.format(id=allrecipe_id)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    response = requests.get(url, headers=headers)
+
+    food_dict = {"allrecipe_id": allrecipe_id}
+
+    if response.status_code != 200:
+        logger.warn(
+            "Bad response from allrecipe", extra={"response": response}
+        )
+        return food_dict
+
+    import pdb
+
+    pdb.set_trace()
+    soup = BeautifulSoup(response.text, "html.parser")
+    food_dict["title"] = get_title_from_soup(soup)
+    food_dict["description"] = get_description_from_soup(soup)
+    food_dict["allrecipe_rating"] = get_rating_from_soup(soup)
+
+    return food_dict