Bump version to 0.10.0

Procfile

@@ -0,0 +1,2 @@
+web: python manage.py runserver 0.0.0.0:8014
+worker: celery -A vrobbler worker -l DEBUG

poetry.lock

@@ -587,6 +587,20 @@ category = "main"
 optional = false
 python-versions = ">=3.7"
 
+[[package]]
+name = "honcho"
+version = "1.1.0"
+description = "Honcho: a Python clone of Foreman. For managing Procfile-based applications."
+category = "main"
+optional = false
+python-versions = "*"
+
+[package.dependencies]
+colorama = {version = "*", markers = "sys_platform == \"win32\""}
+
+[package.extras]
+export = ["jinja2 (>=2.7,<3)"]
+
 [[package]]
 name = "httpcore"
 version = "0.16.3"
@@ -1598,7 +1612,7 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "4b71b291b00a768d7d3c253a02faf70249d1f10ba85fcb88fc5c80fecb412332"
+content-hash = "d57d0a79f04c3288d12d8a9fb3579e03fa514d1a130b11c28812150feeb66a06"
 
 [metadata.files]
 amqp = [
@@ -2079,6 +2093,10 @@ h11 = [
     {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"},
     {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
 ]
+honcho = [
+    {file = "honcho-1.1.0-py2.py3-none-any.whl", hash = "sha256:a4d6e3a88a7b51b66351ecfc6e9d79d8f4b87351db9ad7e923f5632cc498122f"},
+    {file = "honcho-1.1.0.tar.gz", hash = "sha256:c5eca0bded4bef6697a23aec0422fd4f6508ea3581979a3485fc4b89357eb2a9"},
+]
 httpcore = [
     {file = "httpcore-0.16.3-py3-none-any.whl", hash = "sha256:da1fb708784a938aa084bde4feb8317056c55037247c787bd7e19eb2c2949dc0"},
     {file = "httpcore-0.16.3.tar.gz", hash = "sha256:c5d6f04e2fc530f39e0c077e6a30caa53f1451096120f1f38b954afd0b17c0cb"},

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "vrobbler"
-version = "0.9.3"
+version = "0.10.0"
 description = ""
 authors = ["Colin Powell <colin@unbl.ink>"]
 
@@ -35,6 +35,7 @@ django-redis = "^5.2.0"
 pylast = "^5.1.0"
 django-encrypted-field = "^1.0.5"
 celery = "^5.2.7"
+honcho = "^1.1.0"
 
 [tool.poetry.dev-dependencies]
 Werkzeug = "2.0.3"

todos.org

@@ -49,6 +49,15 @@ An example of the format:
 311	311	Misdirected Hostility	7	179	S	1740496085	61ff2c1a-fc9c-44c3-8da1-5e50a44245af
 ,
 #+end_src
+* DONE [#B] Allow scrobbling music without MB IDs by grabbing them before scrobble :improvement:
+CLOSED: [2023-02-17 Fri 00:10]
+
+This would allow a few nice flows. One, you'd be able to record the play of an
+entire album by just dropping the muscibrainz_id in. This could be helpful for
+offline listening. It would also mean bad metadata from mopidy would not break
+scrobbling.
+* DONE When updating musicbrainz IDs, clear and run fetch artwrok :improvement:
+CLOSED: [2023-02-17 Fri 00:11]
 * TODO [#A] Add ability to manually scrobble albums or tracks from MB :improvement:
 
 Given a UUID from musicbrainz, we should be able to scrobble an album or
@@ -58,12 +67,6 @@ individual track.
 * TODO [#B] Adjust cancel/finish task to use javascript to submit :improvement:
 * TODO [#B] Implement a detail view for TV shows :improvement:
 * TODO [#B] Implement a detail view for Moviews :improvement:
-* TODO [#B] Allow scrobbling music without MB IDs by grabbing them before scrobble :improvement:
-
-This would allow a few nice flows. One, you'd be able to record the play of an
-entire album by just dropping the muscibrainz_id in. This could be helpful for
-offline listening. It would also mean bad metadata from mopidy would not break
-scrobbling.
 * TODO [#C] Implement keeping track of week/month/year chart-toppers :improvement:
 :LOGBOOK:
 CLOCK: [2023-01-30 Mon 16:30]--[2023-01-30 Mon 18:00] =>  1:30
@@ -372,9 +375,8 @@ has to re-populate when the server restarts.
 }
 #+end_src
 * TODO [#C] Consider a purge command for duplicated and stuck in-progress scrobbles :improvement:
-* TODO Figure out how to add to web-scrobbler :imropvement:
+* TODO [#C] Figure out how to add to web-scrobbler :imropvement:
 
 An example:
 https://github.com/web-scrobbler/web-scrobbler/blob/master/src/core/background/scrobbler/maloja-scrobbler.js
 
-* TODO When updating musicbrainz IDs, clear and run fetch artwrok :improvement:

vrobbler/apps/books/admin.py

@@ -0,0 +1,25 @@
+from django.contrib import admin
+
+from books.models import Author, Book
+
+from scrobbles.admin import ScrobbleInline
+
+
+@admin.register(Author)
+class AlbumAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = ("name", "openlibrary_id")
+    ordering = ("name",)
+
+
+@admin.register(Book)
+class ArtistAdmin(admin.ModelAdmin):
+    date_hierarchy = "created"
+    list_display = (
+        "title",
+        "isbn",
+        "first_publish_year",
+        "pages",
+        "openlibrary_id",
+    )
+    ordering = ("title",)

vrobbler/apps/books/migrations/0001_initial.py

@@ -0,0 +1,128 @@
+# Generated by Django 4.1.5 on 2023-02-19 20:17
+
+from django.db import migrations, models
+import django_extensions.db.fields
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name='Author',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('name', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+            ],
+            options={
+                'get_latest_by': 'modified',
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Book',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                (
+                    'uuid',
+                    models.UUIDField(
+                        blank=True,
+                        default=uuid.uuid4,
+                        editable=False,
+                        null=True,
+                    ),
+                ),
+                (
+                    'run_time',
+                    models.CharField(blank=True, max_length=8, null=True),
+                ),
+                (
+                    'run_time_ticks',
+                    models.PositiveBigIntegerField(blank=True, null=True),
+                ),
+                ('title', models.CharField(max_length=255)),
+                (
+                    'openlibrary_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'goodreads_id',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('koreader_id', models.IntegerField(blank=True, null=True)),
+                (
+                    'koreader_authors',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'koreader_md5',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    'isbn',
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ('pages', models.IntegerField(blank=True, null=True)),
+                (
+                    'language',
+                    models.CharField(blank=True, max_length=4, null=True),
+                ),
+                (
+                    'first_publish_year',
+                    models.IntegerField(blank=True, null=True),
+                ),
+                ('authors', models.ManyToManyField(to='books.author')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/books/models.py

@@ -0,0 +1,68 @@
+import logging
+from typing import Dict
+
+from django.conf import settings
+from django.contrib.auth import get_user_model
+from django.db import models
+from django.urls import reverse
+from django_extensions.db.models import TimeStampedModel
+from scrobbles.mixins import ScrobblableMixin
+
+from vrobbler.apps.books.utils import lookup_book_from_openlibrary
+
+logger = logging.getLogger(__name__)
+User = get_user_model()
+BNULL = {"blank": True, "null": True}
+
+
+class Author(TimeStampedModel):
+    name = models.CharField(max_length=255)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+
+    def __str__(self):
+        return f"{self.name}"
+
+    def fix_metadata(self):
+        logger.warn("Not implemented yet")
+
+
+class Book(ScrobblableMixin):
+    COMPLETION_PERCENT = getattr(settings, 'BOOK_COMPLETION_PERCENT', 95)
+
+    title = models.CharField(max_length=255)
+    authors = models.ManyToManyField(Author)
+    openlibrary_id = models.CharField(max_length=255, **BNULL)
+    goodreads_id = models.CharField(max_length=255, **BNULL)
+    koreader_id = models.IntegerField(**BNULL)
+    koreader_authors = models.CharField(max_length=255, **BNULL)
+    koreader_md5 = models.CharField(max_length=255, **BNULL)
+    isbn = models.CharField(max_length=255, **BNULL)
+    pages = models.IntegerField(**BNULL)
+    language = models.CharField(max_length=4, **BNULL)
+    first_publish_year = models.IntegerField(**BNULL)
+
+    def __str__(self):
+        return f"{self.title} by {self.author}"
+
+    def fix_metadata(self):
+        if not self.openlibrary_id:
+            book_meta = lookup_book_from_openlibrary(self.title, self.author)
+            self.openlibrary_id = book_meta.get("openlibrary_id")
+            self.isbn = book_meta.get("isbn")
+            self.goodreads_id = book_meta.get("goodreads_id")
+            self.first_pubilsh_year = book_meta.get("first_publish_year")
+            self.save()
+
+    @property
+    def author(self):
+        return self.authors.first()
+
+    def get_absolute_url(self):
+        return reverse("books:book_detail", kwargs={'slug': self.uuid})
+
+    @property
+    def pages_for_completion(self) -> int:
+        if not self.pages:
+            logger.warn(f"{self} has no pages, no completion percentage")
+            return 0
+        return int(self.pages * (self.COMPLETION_PERCENT / 100))

vrobbler/apps/books/utils.py

@@ -0,0 +1,47 @@
+import json
+from typing import Optional
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+SEARCH_URL = "https://openlibrary.org/search.json?title={title}"
+ISBN_URL = "https://openlibrary.org/isbn/{isbn}.json"
+
+
+def get_first(key: str, result: dict) -> str:
+    obj = ""
+    if obj_list := result.get(key):
+        obj = obj_list[0]
+    return obj
+
+
+def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
+    search_url = SEARCH_URL.format(title=title)
+    response = requests.get(search_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return {}
+
+    results = json.loads(response.content)
+
+    if len(results.get('docs')) == 0:
+        logger.warn(f"No results found from OL for {title}")
+        return {}
+
+    top = results.get('docs')[0]
+    if author and author not in top['author_name']:
+        logger.warn(
+            f"Lookup for {title} found top result with mismatched author"
+        )
+
+    return {
+        "title": top.get("title"),
+        "isbn": top.get("isbn")[0],
+        "openlibrary_id": top.get("cover_edition_key"),
+        "author_name": get_first("author_name", top),
+        "author_openlibrary_id": get_first("author_key", top),
+        "goodreads_id": get_first("id_goodreads", top),
+        "first_publish_year": top.get("first_publish_year"),
+    }

vrobbler/apps/music/utils.py

@@ -1,5 +1,5 @@
-import re
 import logging
+import re
 
 from scrobbles.musicbrainz import (
     lookup_album_dict_from_mb,
@@ -9,7 +9,7 @@ from scrobbles.musicbrainz import (
 logger = logging.getLogger(__name__)
 
 
-from music.models import Artist, Album, Track
+from music.models import Album, Artist, Track
 
 
 def get_or_create_artist(name: str, mbid: str = None) -> Artist:
@@ -20,6 +20,7 @@ def get_or_create_artist(name: str, mbid: str = None) -> Artist:
         name = re.split("feat.", name, flags=re.IGNORECASE)[0].strip()
     if 'featuring' in name.lower():
         name = re.split("featuring", name, flags=re.IGNORECASE)[0].strip()
+
     artist_dict = lookup_artist_from_mb(name)
     mbid = mbid or artist_dict['id']
 
@@ -30,9 +31,9 @@ def get_or_create_artist(name: str, mbid: str = None) -> Artist:
 
     logger.debug(f"Cleaning artist {name} with {artist_dict['name']}")
     # Clean up bad names in our DB with MB names
-    if artist.name != artist_dict['name']:
-        artist.name = artist_dict["name"]
-        artist.save(update_fields=["name"])
+    # if artist.name != artist_dict["name"]:
+    #    artist.name = artist_dict["name"]
+    #    artist.save(update_fields=["name"])
 
     return artist
 

vrobbler/apps/profiles/models.py

@@ -16,7 +16,7 @@ class UserProfile(TimeStampedModel):
         User, on_delete=models.CASCADE, related_name="profile"
     )
     timezone = models.CharField(
-        max_length=255, choices=PRETTY_TIMEZONE_CHOICES
+        max_length=255, choices=PRETTY_TIMEZONE_CHOICES, default=pytz.UTC
     )
     lastfm_username = models.CharField(max_length=255, **BNULL)
     lastfm_password = EncryptedField(**BNULL)

vrobbler/apps/scrobbles/admin.py

@@ -2,6 +2,7 @@ from django.contrib import admin
 from scrobbles.models import (
     AudioScrobblerTSVImport,
     ChartRecord,
+    KoReaderImport,
     LastFmImport,
     Scrobble,
 )
@@ -14,15 +15,7 @@ class ScrobbleInline(admin.TabularInline):
     exclude = ('source_id', 'scrobble_log')
 
 
-@admin.register(AudioScrobblerTSVImport)
-class AudioScrobblerTSVImportAdmin(admin.ModelAdmin):
-    date_hierarchy = "created"
-    list_display = ("uuid", "created", "process_count", "tsv_file")
-    ordering = ("-created",)
-
-
-@admin.register(LastFmImport)
-class LastFmImportAdmin(admin.ModelAdmin):
+class ImportBaseAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
     list_display = (
         "uuid",
@@ -33,6 +26,21 @@ class LastFmImportAdmin(admin.ModelAdmin):
     ordering = ("-created",)
 
 
+@admin.register(AudioScrobblerTSVImport)
+class AudioScrobblerTSVImportAdmin(ImportBaseAdmin):
+    """"""
+
+
+@admin.register(LastFmImport)
+class LastFmImportAdmin(ImportBaseAdmin):
+    """"""
+
+
+@admin.register(KoReaderImport)
+class KoReaderImportAdmin(ImportBaseAdmin):
+    """"""
+
+
 @admin.register(ChartRecord)
 class ChartRecordAdmin(admin.ModelAdmin):
     date_hierarchy = "created"
@@ -71,21 +79,21 @@ class ScrobbleAdmin(admin.ModelAdmin):
         "is_paused",
         "played_to_completion",
     )
-    raw_id_fields = ('video', 'podcast_episode', 'track', 'sport_event')
+    raw_id_fields = (
+        'video',
+        'podcast_episode',
+        'track',
+        'sport_event',
+        'book',
+    )
     list_filter = ("is_paused", "in_progress", "source", "track__artist")
     ordering = ("-timestamp",)
 
     def media_name(self, obj):
-        if obj.video:
-            return obj.video
-        if obj.track:
-            return obj.track
-        if obj.podcast_episode:
-            return obj.podcast_episode
-        if obj.sport_event:
-            return obj.sport_event
+        return obj.media_obj
 
     def media_type(self, obj):
+        return obj.media_obj.__class__.__name__
         if obj.video:
             return "Video"
         if obj.track:

vrobbler/apps/scrobbles/koreader.py

@@ -0,0 +1,124 @@
+import logging
+from datetime import datetime
+import sqlite3
+from enum import Enum
+
+import pytz
+
+from books.models import Author, Book
+from scrobbles.models import Scrobble
+from django.utils import timezone
+
+logger = logging.getLogger(__name__)
+
+
+class KoReaderBookColumn(Enum):
+    ID = 0
+    TITLE = 1
+    AUTHORS = 2
+    NOTES = 3
+    LAST_OPEN = 4
+    HIGHLIGHTS = 5
+    PAGES = 6
+    SERIES = 7
+    LANGUAGE = 8
+    MD5 = 9
+    TOTAL_READ_TIME = 10
+    TOTAL_READ_PAGES = 11
+
+
+class KoReaderPageStatColumn(Enum):
+    ID_BOOK = 0
+    PAGE = 1
+    START_TIME = 2
+    DURATION = 3
+    TOTAL_PAGES = 4
+
+
+def process_koreader_sqlite_file(sqlite_file_path, user_id):
+    """Given a sqlite file from KoReader, open the book table, iterate
+    over rows creating scrobbles from each book found"""
+    # Create a SQL connection to our SQLite database
+    con = sqlite3.connect(sqlite_file_path)
+    cur = con.cursor()
+
+    # Return all results of query
+    book_table = cur.execute("SELECT * FROM book")
+    new_scrobbles = []
+    for book_row in book_table:
+        authors = book_row[KoReaderBookColumn.AUTHORS.value].split('\n')
+        author_list = []
+        for author_str in authors:
+            logger.debug(f"Looking up author {author_str}")
+
+            if author_str == "N/A":
+                continue
+
+            author, created = Author.objects.get_or_create(name=author_str)
+            if created:
+                author.fix_metadata()
+            author_list.append(author)
+            logger.debug(f"Found author {author}, created: {created}")
+
+        book, created = Book.objects.get_or_create(
+            koreader_md5=book_row[KoReaderBookColumn.MD5.value]
+        )
+
+        if created:
+            book.title = book_row[KoReaderBookColumn.TITLE.value]
+            book.pages = book_row[KoReaderBookColumn.PAGES.value]
+            book.koreader_id = int(book_row[KoReaderBookColumn.ID.value])
+            book.koreader_authors = book_row[KoReaderBookColumn.AUTHORS.value]
+            book.run_time_ticks = int(book_row[KoReaderBookColumn.PAGES.value])
+            book.save(
+                update_fields=[
+                    "title",
+                    "pages",
+                    "koreader_id",
+                    "koreader_authors",
+                ]
+            )
+            book.fix_metadata()
+            if author_list:
+                book.authors.add(*[a.id for a in author_list])
+
+        playback_position = int(
+            book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]
+        )
+        playback_position_ticks = playback_position * 1000
+        pages_read = int(book_row[KoReaderBookColumn.TOTAL_READ_PAGES.value])
+        timestamp = datetime.utcfromtimestamp(
+            book_row[KoReaderBookColumn.LAST_OPEN.value]
+        ).replace(tzinfo=pytz.utc)
+
+        new_scrobble = Scrobble(
+            book_id=book.id,
+            user_id=user_id,
+            source="KOReader",
+            timestamp=timestamp,
+            playback_position_ticks=playback_position_ticks,
+            playback_position=playback_position,
+            played_to_completion=True,
+            in_progress=False,
+            book_pages_read=pages_read,
+        )
+
+        existing = Scrobble.objects.filter(
+            timestamp=timestamp, book=book
+        ).first()
+        if existing:
+            logger.debug(f"Skipping existing scrobble {new_scrobble}")
+            continue
+
+        logger.debug(f"Queued scrobble {new_scrobble} for creation")
+        new_scrobbles.append(new_scrobble)
+
+    # Be sure to close the connection
+    con.close()
+
+    created = Scrobble.objects.bulk_create(new_scrobbles)
+    logger.info(
+        f"Created {len(created)} scrobbles",
+        extra={'created_scrobbles': created},
+    )
+    return created

vrobbler/apps/scrobbles/lastfm.py

@@ -47,17 +47,17 @@ class LastFM:
         new_scrobbles = []
         source = "Last.fm"
         source_id = ""
-        latest_scrobbles = self.get_last_scrobbles(time_from=last_processed)
+        lastfm_scrobbles = self.get_last_scrobbles(time_from=last_processed)
 
-        for scrobble in latest_scrobbles:
-            timestamp = scrobble.pop('timestamp')
+        for lfm_scrobble in lastfm_scrobbles:
+            timestamp = lfm_scrobble.pop('timestamp')
 
-            artist = get_or_create_artist(scrobble.pop('artist'))
-            album = get_or_create_album(scrobble.pop('album'), artist)
+            artist = get_or_create_artist(lfm_scrobble.pop('artist'))
+            album = get_or_create_album(lfm_scrobble.pop('album'), artist)
 
-            scrobble['artist'] = artist
-            scrobble['album'] = album
-            track = get_or_create_track(**scrobble)
+            lfm_scrobble['artist'] = artist
+            lfm_scrobble['album'] = album
+            track = get_or_create_track(**lfm_scrobble)
 
             new_scrobble = Scrobble(
                 user=self.vrobbler_user,
@@ -89,27 +89,6 @@ class LastFM:
         )
         return created
 
-    @staticmethod
-    def undo_lastfm_import(process_log, dryrun=True):
-        """Given a newline separated list of scrobbles, delete them"""
-        from scrobbles.models import Scrobble
-
-        if not process_log:
-            logger.warning("No lines in process log found to undo")
-            return
-
-        for line in process_log.split('\n'):
-            scrobble_id = line.split("\t")[0]
-            scrobble = Scrobble.objects.filter(id=scrobble_id).first()
-            if not scrobble:
-                logger.warning(
-                    f"Could not find scrobble {scrobble_id} to undo"
-                )
-                continue
-            logger.info(f"Removing scrobble {scrobble_id}")
-            if not dryrun:
-                scrobble.delete()
-
     def get_last_scrobbles(self, time_from=None, time_to=None):
         """Given a user, Last.fm api key, and secret key, grab a list of scrobbled
         tracks"""
@@ -144,8 +123,8 @@ class LastFM:
                     "LastFM barfed trying to get the track for {scrobble.track}"
                 )
 
-            if not mbid or not artist:
-                logger.warn(f"Silly LastFM, bad data, bailing on {scrobble}")
+            if not artist:
+                logger.warn(f"Silly LastFM, no artist found for {scrobble}")
                 continue
 
             timestamp = datetime.utcfromtimestamp(

vrobbler/apps/scrobbles/migrations/0020_alter_audioscrobblertsvimport_options_and_more.py

@@ -0,0 +1,97 @@
+# Generated by Django 4.1.5 on 2023-02-19 03:53
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import scrobbles.models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        (
+            'scrobbles',
+            '0019_rename_processed_on_lastfmimport_processed_finished_and_more',
+        ),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='audioscrobblertsvimport',
+            options={},
+        ),
+        migrations.AlterModelOptions(
+            name='lastfmimport',
+            options={},
+        ),
+        migrations.RenameField(
+            model_name='audioscrobblertsvimport',
+            old_name='processed_on',
+            new_name='processed_finished',
+        ),
+        migrations.AddField(
+            model_name='audioscrobblertsvimport',
+            name='processing_started',
+            field=models.DateTimeField(blank=True, null=True),
+        ),
+        migrations.CreateModel(
+            name='KoReaderImport',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name='ID',
+                    ),
+                ),
+                (
+                    'created',
+                    django_extensions.db.fields.CreationDateTimeField(
+                        auto_now_add=True, verbose_name='created'
+                    ),
+                ),
+                (
+                    'modified',
+                    django_extensions.db.fields.ModificationDateTimeField(
+                        auto_now=True, verbose_name='modified'
+                    ),
+                ),
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)),
+                (
+                    'processing_started',
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                (
+                    'processed_finished',
+                    models.DateTimeField(blank=True, null=True),
+                ),
+                ('process_log', models.TextField(blank=True, null=True)),
+                ('process_count', models.IntegerField(blank=True, null=True)),
+                (
+                    'sqlite_file',
+                    models.FileField(
+                        blank=True,
+                        null=True,
+                        upload_to=scrobbles.models.KoReaderImport.get_path,
+                    ),
+                ),
+                (
+                    'user',
+                    models.ForeignKey(
+                        blank=True,
+                        null=True,
+                        on_delete=django.db.models.deletion.DO_NOTHING,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0021_scrobble_book.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.1.5 on 2023-02-19 20:20
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('books', '0001_initial'),
+        ('scrobbles', '0020_alter_audioscrobblertsvimport_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scrobble',
+            name='book',
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.DO_NOTHING,
+                to='books.book',
+            ),
+        ),
+    ]

vrobbler/apps/scrobbles/migrations/0022_scrobble_book_pages_read.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-02-20 00:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scrobbles', '0021_scrobble_book'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='scrobble',
+            name='book_pages_read',
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/scrobbles/models.py

@@ -7,6 +7,7 @@ from django.db import models
 from django.utils import timezone
 from django_extensions.db.models import TimeStampedModel
 from music.models import Artist, Track
+from books.models import Book
 from podcasts.models import Episode
 from profiles.utils import now_user_timezone
 from scrobbles.lastfm import LastFM
@@ -19,62 +20,7 @@ User = get_user_model()
 BNULL = {"blank": True, "null": True}
 
 
-class AudioScrobblerTSVImport(TimeStampedModel):
-    def get_path(instance, filename):
-        extension = filename.split('.')[-1]
-        uuid = instance.uuid
-        return f'audioscrobbler-uploads/{uuid}.{extension}'
-
-    user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
-    uuid = models.UUIDField(editable=False, default=uuid4)
-    tsv_file = models.FileField(upload_to=get_path, **BNULL)
-    processed_on = models.DateTimeField(**BNULL)
-    process_log = models.TextField(**BNULL)
-    process_count = models.IntegerField(**BNULL)
-
-    def __str__(self):
-        if self.tsv_file:
-            return f"Audioscrobbler TSV upload: {self.tsv_file.path}"
-        return f"Audioscrobbler TSV upload {self.id}"
-
-    def process(self, force=False):
-        from scrobbles.tsv import process_audioscrobbler_tsv_file
-
-        if self.processed_on and not force:
-            logger.info(f"{self} already processed on {self.processed_on}")
-            return
-
-        tz = None
-        if self.user:
-            tz = self.user.profile.tzinfo
-        scrobbles = process_audioscrobbler_tsv_file(
-            self.tsv_file.path, user_tz=tz
-        )
-        self.process_log = ""
-        if scrobbles:
-            for count, scrobble in enumerate(scrobbles):
-                scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}"
-                log_line = f"{scrobble_str}"
-                if count > 0:
-                    log_line = "\n" + log_line
-                self.process_log += log_line
-            self.process_count = len(scrobbles)
-        else:
-            self.process_log = f"Created no new scrobbles"
-            self.process_count = 0
-
-        self.processed_on = timezone.now()
-        self.save(
-            update_fields=['processed_on', 'process_count', 'process_log']
-        )
-
-    def undo(self, dryrun=True):
-        from scrobbles.tsv import undo_audioscrobbler_tsv_import
-
-        undo_audioscrobbler_tsv_import(self.process_log, dryrun)
-
-
-class LastFmImport(TimeStampedModel):
+class BaseFileImportMixin(TimeStampedModel):
     user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL)
     uuid = models.UUIDField(editable=False, default=uuid4)
     processing_started = models.DateTimeField(**BNULL)
@@ -82,8 +28,135 @@ class LastFmImport(TimeStampedModel):
     process_log = models.TextField(**BNULL)
     process_count = models.IntegerField(**BNULL)
 
+    class Meta:
+        abstract = True
+
     def __str__(self):
-        return f"LastFM Import: {self.uuid}"
+        return f"Scrobble import {self.id}"
+
+    def process(self, force=False):
+        logger.warning("Process not implemented")
+
+    def undo(self, dryrun=False):
+        """Accepts the log from a scrobble import and removes the scrobbles"""
+        from scrobbles.models import Scrobble
+
+        if not self.process_log:
+            logger.warning("No lines in process log found to undo")
+            return
+
+        for line in self.process_log.split('\n'):
+            scrobble_id = line.split("\t")[0]
+            scrobble = Scrobble.objects.filter(id=scrobble_id).first()
+            if not scrobble:
+                logger.warning(
+                    f"Could not find scrobble {scrobble_id} to undo"
+                )
+                continue
+            logger.info(f"Removing scrobble {scrobble_id}")
+            if not dryrun:
+                scrobble.delete()
+        self.processed_finished = None
+        self.processing_started = None
+        self.process_count = None
+        self.process_log = ""
+        self.save(
+            update_fields=[
+                "processed_finished",
+                "processing_started",
+                "process_log",
+                "process_count",
+            ]
+        )
+
+    def mark_started(self):
+        self.processing_started = timezone.now()
+        self.save(update_fields=["processing_started"])
+
+    def mark_finished(self):
+        self.processed_finished = timezone.now()
+        self.save(update_fields=['processed_finished'])
+
+    def record_log(self, scrobbles):
+        self.process_log = ""
+        if not scrobbles:
+            self.process_count = 0
+            self.save(update_fields=["process_log", "process_count"])
+            return
+
+        for count, scrobble in enumerate(scrobbles):
+            scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.media_obj.title}"
+            log_line = f"{scrobble_str}"
+            if count > 0:
+                log_line = "\n" + log_line
+            self.process_log += log_line
+        self.process_count = len(scrobbles)
+        self.save(update_fields=["process_log", "process_count"])
+
+
+class KoReaderImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "KOReader Import"
+
+    def get_path(instance, filename):
+        extension = filename.split('.')[-1]
+        uuid = instance.uuid
+        return f'koreader-uploads/{uuid}.{extension}'
+
+    sqlite_file = models.FileField(upload_to=get_path, **BNULL)
+
+    def process(self, force=False):
+        from scrobbles.koreader import process_koreader_sqlite_file
+
+        if self.processed_finished and not force:
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
+            return
+
+        self.mark_started()
+        scrobbles = process_koreader_sqlite_file(
+            self.sqlite_file.path, self.user.id
+        )
+        self.record_log(scrobbles)
+        self.mark_finished()
+
+
+class AudioScrobblerTSVImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "AudioScrobbler TSV Import"
+
+    def get_path(instance, filename):
+        extension = filename.split('.')[-1]
+        uuid = instance.uuid
+        return f'audioscrobbler-uploads/{uuid}.{extension}'
+
+    tsv_file = models.FileField(upload_to=get_path, **BNULL)
+
+    def process(self, force=False):
+        from scrobbles.tsv import process_audioscrobbler_tsv_file
+
+        if self.processed_finished and not force:
+            logger.info(
+                f"{self} already processed on {self.processed_finished}"
+            )
+            return
+
+        self.mark_started()
+
+        tz = None
+        if self.user:
+            tz = self.user.profile.tzinfo
+        scrobbles = process_audioscrobbler_tsv_file(
+            self.tsv_file.path, self.user.id, user_tz=tz
+        )
+        self.record_log(scrobbles)
+        self.mark_finished()
+
+
+class LastFmImport(BaseFileImportMixin):
+    class Meta:
+        verbose_name = "Last.FM Import"
 
     def process(self, import_all=False):
         """Import scrobbles found on LastFM"""
@@ -111,36 +184,12 @@ class LastFmImport(TimeStampedModel):
         if last_import:
             last_processed = last_import.processed_finished
 
-        self.processing_started = timezone.now()
-        self.save(update_fields=['processing_started'])
+        self.mark_started()
 
         scrobbles = lastfm.import_from_lastfm(last_processed)
-        self.process_log = ""
-        if scrobbles:
-            for count, scrobble in enumerate(scrobbles):
-                scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}"
-                log_line = f"{scrobble_str}"
-                if count > 0:
-                    log_line = "\n" + log_line
-                self.process_log += log_line
-            self.process_count = len(scrobbles)
-        else:
-            self.process_count = 0
 
-        self.processed_finished = timezone.now()
-        self.save(
-            update_fields=[
-                'processed_finished',
-                'process_count',
-                'process_log',
-            ]
-        )
-
-    def undo(self, dryrun=False):
-        """Undo import of scrobbles from LastFM"""
-        LastFM.undo_lastfm_import(self.process_log, dryrun)
-        self.processed_finished = None
-        self.save(update_fields=['processed_finished'])
+        self.record_log(scrobbles)
+        self.mark_finished()
 
 
 class ChartRecord(TimeStampedModel):
@@ -231,19 +280,29 @@ class Scrobble(TimeStampedModel):
     sport_event = models.ForeignKey(
         SportEvent, on_delete=models.DO_NOTHING, **BNULL
     )
+    book = models.ForeignKey(Book, on_delete=models.DO_NOTHING, **BNULL)
     user = models.ForeignKey(
         User, blank=True, null=True, on_delete=models.DO_NOTHING
     )
+
+    # Time keeping
     timestamp = models.DateTimeField(**BNULL)
     playback_position_ticks = models.PositiveBigIntegerField(**BNULL)
     playback_position = models.CharField(max_length=8, **BNULL)
+
+    # Status indicators
     is_paused = models.BooleanField(default=False)
     played_to_completion = models.BooleanField(default=False)
+    in_progress = models.BooleanField(default=True)
+
+    # Metadata
     source = models.CharField(max_length=255, **BNULL)
     source_id = models.TextField(**BNULL)
-    in_progress = models.BooleanField(default=True)
     scrobble_log = models.TextField(**BNULL)
 
+    # Fields for keeping track of reads between scrobbles
+    book_pages_read = models.IntegerField(**BNULL)
+
     def save(self, *args, **kwargs):
         if not self.uuid:
             self.uuid = uuid4()
@@ -272,7 +331,10 @@ class Scrobble(TimeStampedModel):
 
     @property
     def percent_played(self) -> int:
-        if not self.media_obj.run_time_ticks:
+        if not self.media_obj:
+            return 0
+
+        if self.media_obj and not self.media_obj.run_time_ticks:
             return 100
 
         if not self.playback_position_ticks and self.played_to_completion:
@@ -309,6 +371,8 @@ class Scrobble(TimeStampedModel):
             media_obj = self.podcast_episode
         if self.sport_event:
             media_obj = self.sport_event
+        if self.book:
+            media_obj = self.book
         return media_obj
 
     def __str__(self):
@@ -332,6 +396,9 @@ class Scrobble(TimeStampedModel):
         if media.__class__.__name__ == 'SportEvent':
             media_query = models.Q(sport_event=media)
             scrobble_data['sport_event_id'] = media.id
+        if media.__class__.__name__ == 'Book':
+            media_query = models.Q(book=media)
+            scrobble_data['book_id'] = media.id
 
         scrobble = (
             cls.objects.filter(

vrobbler/apps/scrobbles/tasks.py

@@ -1,7 +1,11 @@
 import logging
 from celery import shared_task
 
-from scrobbles.models import AudioScrobblerTSVImport, LastFmImport
+from scrobbles.models import (
+    AudioScrobblerTSVImport,
+    KoReaderImport,
+    LastFmImport,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -22,3 +26,12 @@ def process_tsv_import(import_id):
         logger.warn(f"AudioScrobblerTSVImport not found with id {import_id}")
 
     tsv_import.process()
+
+
+@shared_task
+def process_koreader_import(import_id):
+    koreader_import = KoReaderImport.objects.filter(id=import_id).first()
+    if not koreader_import:
+        logger.warn(f"KOReaderImport not found with id {import_id}")
+
+    koreader_import.process()

vrobbler/apps/scrobbles/tsv.py

@@ -13,7 +13,7 @@ from music.utils import (
 logger = logging.getLogger(__name__)
 
 
-def process_audioscrobbler_tsv_file(file_path, user_tz=None):
+def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None):
     """Takes a path to a file of TSV data and imports it as past scrobbles"""
     new_scrobbles = []
     if not user_tz:
@@ -49,12 +49,13 @@ def process_audioscrobbler_tsv_file(file_path, user_tz=None):
             )
 
             timestamp = (
-                datetime.fromtimestamp(int(row[6]))
+                datetime.utcfromtimestamp(int(row[6]))
                 .replace(tzinfo=user_tz)
                 .astimezone(pytz.utc)
             )
 
             new_scrobble = Scrobble(
+                user_id=user_id,
                 timestamp=timestamp,
                 source=source,
                 source_id=source_id,
@@ -77,20 +78,3 @@ def process_audioscrobbler_tsv_file(file_path, user_tz=None):
             extra={'created_scrobbles': created},
         )
         return created
-
-
-def undo_audioscrobbler_tsv_import(process_log, dryrun=True):
-    """Accepts the log from a TSV import and removes the scrobbles"""
-    if not process_log:
-        logger.warning("No lines in process log found to undo")
-        return
-
-    for line in process_log.split('\n'):
-        scrobble_id = line.split("\t")[0]
-        scrobble = Scrobble.objects.filter(id=scrobble_id).first()
-        if not scrobble:
-            logger.warning(f"Could not find scrobble {scrobble_id} to undo")
-            continue
-        logger.info(f"Removing scrobble {scrobble_id}")
-        if not dryrun:
-            scrobble.delete()

vrobbler/apps/scrobbles/views.py

@@ -35,7 +35,12 @@ from scrobbles.constants import (
 from scrobbles.export import export_scrobbles
 from scrobbles.forms import ExportScrobbleForm, ScrobbleForm
 from scrobbles.imdb import lookup_video_from_imdb
-from scrobbles.models import AudioScrobblerTSVImport, LastFmImport, Scrobble
+from scrobbles.models import (
+    AudioScrobblerTSVImport,
+    KoReaderImport,
+    LastFmImport,
+    Scrobble,
+)
 from scrobbles.scrobblers import (
     jellyfin_scrobble_track,
     jellyfin_scrobble_video,
@@ -48,7 +53,11 @@ from scrobbles.serializers import (
     AudioScrobblerTSVImportSerializer,
     ScrobbleSerializer,
 )
-from scrobbles.tasks import process_lastfm_import, process_tsv_import
+from scrobbles.tasks import (
+    process_koreader_import,
+    process_lastfm_import,
+    process_tsv_import,
+)
 from scrobbles.thesportsdb import lookup_event_from_thesportsdb
 
 logger = logging.getLogger(__name__)
@@ -177,6 +186,22 @@ class AudioScrobblerImportCreateView(
         return HttpResponseRedirect(self.get_success_url())
 
 
+class KoReaderImportCreateView(
+    LoginRequiredMixin, JsonableResponseMixin, CreateView
+):
+    model = KoReaderImport
+    fields = ['sqlite_file']
+    template_name = 'scrobbles/upload_form.html'
+    success_url = reverse_lazy('vrobbler-home')
+
+    def form_valid(self, form):
+        self.object = form.save(commit=False)
+        self.object.user = self.request.user
+        self.object.save()
+        process_koreader_import.delay(self.object.id)
+        return HttpResponseRedirect(self.get_success_url())
+
+
 @permission_classes([IsAuthenticated])
 @api_view(['GET'])
 def lastfm_import(request):

vrobbler/settings.py

@@ -97,6 +97,7 @@ INSTALLED_APPS = [
     "music",
     "podcasts",
     "sports",
+    "books",
     "mathfilters",
     "rest_framework",
     "allauth",

vrobbler/templates/scrobbles/scrobble_list.html

@@ -326,7 +326,18 @@
                 </div>
                 <div class="modal-footer">
                     <button type="submit" class="btn btn-primary">Import</button>
-                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
+                </div>
+            </form>
+            <form action="{% url 'koreader-file-upload' %}" method="post" enctype="multipart/form-data">
+                <div class="modal-body">
+                    {% csrf_token %}
+                    <div class="form-group">
+                        <label for="tsv_file" class="col-form-label">KOReader sqlite file:</label>
+                        <input type="file" name="sqlite_file" class="form-control" id="id_sqlite_file">
+                    </div>
+                </div>
+                <div class="modal-footer">
+                    <button type="submit" class="btn btn-primary">Import</button>
                 </div>
             </form>
         </div>

vrobbler/urls.py

@@ -24,6 +24,11 @@ urlpatterns = [
         scrobbles_views.AudioScrobblerImportCreateView.as_view(),
         name='audioscrobbler-file-upload',
     ),
+    path(
+        'manual/koreader/',
+        scrobbles_views.KoReaderImportCreateView.as_view(),
+        name='koreader-file-upload',
+    ),
     path("", include(music_urls, namespace="music")),
     path("", include(video_urls, namespace="videos")),
     path(