diff --git a/vrobbler/apps/books/admin.py b/vrobbler/apps/books/admin.py new file mode 100644 index 0000000..fc7a6bf --- /dev/null +++ b/vrobbler/apps/books/admin.py @@ -0,0 +1,25 @@ +from django.contrib import admin + +from books.models import Author, Book + +from scrobbles.admin import ScrobbleInline + + +@admin.register(Author) +class AlbumAdmin(admin.ModelAdmin): + date_hierarchy = "created" + list_display = ("name", "openlibrary_id") + ordering = ("name",) + + +@admin.register(Book) +class ArtistAdmin(admin.ModelAdmin): + date_hierarchy = "created" + list_display = ( + "title", + "isbn", + "first_publish_year", + "pages", + "openlibrary_id", + ) + ordering = ("title",) diff --git a/vrobbler/apps/books/migrations/0001_initial.py b/vrobbler/apps/books/migrations/0001_initial.py new file mode 100644 index 0000000..ebecbef --- /dev/null +++ b/vrobbler/apps/books/migrations/0001_initial.py @@ -0,0 +1,128 @@ +# Generated by Django 4.1.5 on 2023-02-19 20:17 + +from django.db import migrations, models +import django_extensions.db.fields +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name='Author', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ( + 'created', + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name='created' + ), + ), + ( + 'modified', + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name='modified' + ), + ), + ('name', models.CharField(max_length=255)), + ( + 'openlibrary_id', + models.CharField(blank=True, max_length=255, null=True), + ), + ], + options={ + 'get_latest_by': 'modified', + 'abstract': False, + }, + ), + migrations.CreateModel( + name='Book', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ( + 'created', + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name='created' + ), + ), + ( + 'modified', + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name='modified' + ), + ), + ( + 'uuid', + models.UUIDField( + blank=True, + default=uuid.uuid4, + editable=False, + null=True, + ), + ), + ( + 'run_time', + models.CharField(blank=True, max_length=8, null=True), + ), + ( + 'run_time_ticks', + models.PositiveBigIntegerField(blank=True, null=True), + ), + ('title', models.CharField(max_length=255)), + ( + 'openlibrary_id', + models.CharField(blank=True, max_length=255, null=True), + ), + ( + 'goodreads_id', + models.CharField(blank=True, max_length=255, null=True), + ), + ('koreader_id', models.IntegerField(blank=True, null=True)), + ( + 'koreader_authors', + models.CharField(blank=True, max_length=255, null=True), + ), + ( + 'koreader_md5', + models.CharField(blank=True, max_length=255, null=True), + ), + ( + 'isbn', + models.CharField(blank=True, max_length=255, null=True), + ), + ('pages', models.IntegerField(blank=True, null=True)), + ( + 'language', + models.CharField(blank=True, max_length=4, null=True), + ), + ( + 'first_publish_year', + models.IntegerField(blank=True, null=True), + ), + ('authors', models.ManyToManyField(to='books.author')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/vrobbler/apps/books/migrations/__init__.py b/vrobbler/apps/books/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vrobbler/apps/books/models.py b/vrobbler/apps/books/models.py new file mode 100644 index 0000000..48fda37 --- /dev/null +++ b/vrobbler/apps/books/models.py @@ -0,0 +1,68 @@ +import logging +from typing import Dict + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.db import models +from django.urls import reverse +from django_extensions.db.models import TimeStampedModel +from scrobbles.mixins import ScrobblableMixin + +from vrobbler.apps.books.utils import lookup_book_from_openlibrary + +logger = logging.getLogger(__name__) +User = get_user_model() +BNULL = {"blank": True, "null": True} + + +class Author(TimeStampedModel): + name = models.CharField(max_length=255) + openlibrary_id = models.CharField(max_length=255, **BNULL) + + def __str__(self): + return f"{self.name}" + + def fix_metadata(self): + logger.warn("Not implemented yet") + + +class Book(ScrobblableMixin): + COMPLETION_PERCENT = getattr(settings, 'BOOK_COMPLETION_PERCENT', 95) + + title = models.CharField(max_length=255) + authors = models.ManyToManyField(Author) + openlibrary_id = models.CharField(max_length=255, **BNULL) + goodreads_id = models.CharField(max_length=255, **BNULL) + koreader_id = models.IntegerField(**BNULL) + koreader_authors = models.CharField(max_length=255, **BNULL) + koreader_md5 = models.CharField(max_length=255, **BNULL) + isbn = models.CharField(max_length=255, **BNULL) + pages = models.IntegerField(**BNULL) + language = models.CharField(max_length=4, **BNULL) + first_publish_year = models.IntegerField(**BNULL) + + def __str__(self): + return f"{self.title} by {self.author}" + + def fix_metadata(self): + if not self.openlibrary_id: + book_meta = lookup_book_from_openlibrary(self.title, self.author) + self.openlibrary_id = book_meta.get("openlibrary_id") + self.isbn = book_meta.get("isbn") + self.goodreads_id = book_meta.get("goodreads_id") + self.first_pubilsh_year = book_meta.get("first_publish_year") + self.save() + + @property + def author(self): + return self.authors.first() + + def get_absolute_url(self): + return reverse("books:book_detail", kwargs={'slug': self.uuid}) + + @property + def pages_for_completion(self) -> int: + if not self.pages: + logger.warn(f"{self} has no pages, no completion percentage") + return 0 + return int(self.pages * (self.COMPLETION_PERCENT / 100)) diff --git a/vrobbler/apps/books/utils.py b/vrobbler/apps/books/utils.py new file mode 100644 index 0000000..16ef6cd --- /dev/null +++ b/vrobbler/apps/books/utils.py @@ -0,0 +1,47 @@ +import json +from typing import Optional +import requests +import logging + +logger = logging.getLogger(__name__) + +SEARCH_URL = "https://openlibrary.org/search.json?title={title}" +ISBN_URL = "https://openlibrary.org/isbn/{isbn}.json" + + +def get_first(key: str, result: dict) -> str: + obj = "" + if obj_list := result.get(key): + obj = obj_list[0] + return obj + + +def lookup_book_from_openlibrary(title: str, author: str = None) -> dict: + search_url = SEARCH_URL.format(title=title) + response = requests.get(search_url) + + if response.status_code != 200: + logger.warn(f"Bad response from OL: {response.status_code}") + return {} + + results = json.loads(response.content) + + if len(results.get('docs')) == 0: + logger.warn(f"No results found from OL for {title}") + return {} + + top = results.get('docs')[0] + if author and author not in top['author_name']: + logger.warn( + f"Lookup for {title} found top result with mismatched author" + ) + + return { + "title": top.get("title"), + "isbn": top.get("isbn")[0], + "openlibrary_id": top.get("cover_edition_key"), + "author_name": get_first("author_name", top), + "author_openlibrary_id": get_first("author_key", top), + "goodreads_id": get_first("id_goodreads", top), + "first_publish_year": top.get("first_publish_year"), + } diff --git a/vrobbler/apps/music/utils.py b/vrobbler/apps/music/utils.py index 106820e..dbf016a 100644 --- a/vrobbler/apps/music/utils.py +++ b/vrobbler/apps/music/utils.py @@ -1,5 +1,5 @@ -import re import logging +import re from scrobbles.musicbrainz import ( lookup_album_dict_from_mb, @@ -9,7 +9,7 @@ from scrobbles.musicbrainz import ( logger = logging.getLogger(__name__) -from music.models import Artist, Album, Track +from music.models import Album, Artist, Track def get_or_create_artist(name: str, mbid: str = None) -> Artist: diff --git a/vrobbler/apps/profiles/models.py b/vrobbler/apps/profiles/models.py index 8a540fc..d93081d 100644 --- a/vrobbler/apps/profiles/models.py +++ b/vrobbler/apps/profiles/models.py @@ -16,7 +16,7 @@ class UserProfile(TimeStampedModel): User, on_delete=models.CASCADE, related_name="profile" ) timezone = models.CharField( - max_length=255, choices=PRETTY_TIMEZONE_CHOICES + max_length=255, choices=PRETTY_TIMEZONE_CHOICES, default=pytz.UTC ) lastfm_username = models.CharField(max_length=255, **BNULL) lastfm_password = EncryptedField(**BNULL) diff --git a/vrobbler/apps/scrobbles/admin.py b/vrobbler/apps/scrobbles/admin.py index a5df053..07e21a2 100644 --- a/vrobbler/apps/scrobbles/admin.py +++ b/vrobbler/apps/scrobbles/admin.py @@ -2,6 +2,7 @@ from django.contrib import admin from scrobbles.models import ( AudioScrobblerTSVImport, ChartRecord, + KoReaderImport, LastFmImport, Scrobble, ) @@ -14,15 +15,7 @@ class ScrobbleInline(admin.TabularInline): exclude = ('source_id', 'scrobble_log') -@admin.register(AudioScrobblerTSVImport) -class AudioScrobblerTSVImportAdmin(admin.ModelAdmin): - date_hierarchy = "created" - list_display = ("uuid", "created", "process_count", "tsv_file") - ordering = ("-created",) - - -@admin.register(LastFmImport) -class LastFmImportAdmin(admin.ModelAdmin): +class ImportBaseAdmin(admin.ModelAdmin): date_hierarchy = "created" list_display = ( "uuid", @@ -33,6 +26,21 @@ class LastFmImportAdmin(admin.ModelAdmin): ordering = ("-created",) +@admin.register(AudioScrobblerTSVImport) +class AudioScrobblerTSVImportAdmin(ImportBaseAdmin): + """""" + + +@admin.register(LastFmImport) +class LastFmImportAdmin(ImportBaseAdmin): + """""" + + +@admin.register(KoReaderImport) +class KoReaderImportAdmin(ImportBaseAdmin): + """""" + + @admin.register(ChartRecord) class ChartRecordAdmin(admin.ModelAdmin): date_hierarchy = "created" @@ -71,21 +79,21 @@ class ScrobbleAdmin(admin.ModelAdmin): "is_paused", "played_to_completion", ) - raw_id_fields = ('video', 'podcast_episode', 'track', 'sport_event') + raw_id_fields = ( + 'video', + 'podcast_episode', + 'track', + 'sport_event', + 'book', + ) list_filter = ("is_paused", "in_progress", "source", "track__artist") ordering = ("-timestamp",) def media_name(self, obj): - if obj.video: - return obj.video - if obj.track: - return obj.track - if obj.podcast_episode: - return obj.podcast_episode - if obj.sport_event: - return obj.sport_event + return obj.media_obj def media_type(self, obj): + return obj.media_obj.__class__.__name__ if obj.video: return "Video" if obj.track: diff --git a/vrobbler/apps/scrobbles/koreader.py b/vrobbler/apps/scrobbles/koreader.py new file mode 100644 index 0000000..07ee531 --- /dev/null +++ b/vrobbler/apps/scrobbles/koreader.py @@ -0,0 +1,124 @@ +import logging +from datetime import datetime +import sqlite3 +from enum import Enum + +import pytz + +from books.models import Author, Book +from scrobbles.models import Scrobble +from django.utils import timezone + +logger = logging.getLogger(__name__) + + +class KoReaderBookColumn(Enum): + ID = 0 + TITLE = 1 + AUTHORS = 2 + NOTES = 3 + LAST_OPEN = 4 + HIGHLIGHTS = 5 + PAGES = 6 + SERIES = 7 + LANGUAGE = 8 + MD5 = 9 + TOTAL_READ_TIME = 10 + TOTAL_READ_PAGES = 11 + + +class KoReaderPageStatColumn(Enum): + ID_BOOK = 0 + PAGE = 1 + START_TIME = 2 + DURATION = 3 + TOTAL_PAGES = 4 + + +def process_koreader_sqlite_file(sqlite_file_path, user_id): + """Given a sqlite file from KoReader, open the book table, iterate + over rows creating scrobbles from each book found""" + # Create a SQL connection to our SQLite database + con = sqlite3.connect(sqlite_file_path) + cur = con.cursor() + + # Return all results of query + book_table = cur.execute("SELECT * FROM book") + new_scrobbles = [] + for book_row in book_table: + authors = book_row[KoReaderBookColumn.AUTHORS.value].split('\n') + author_list = [] + for author_str in authors: + logger.debug(f"Looking up author {author_str}") + + if author_str == "N/A": + continue + + author, created = Author.objects.get_or_create(name=author_str) + if created: + author.fix_metadata() + author_list.append(author) + logger.debug(f"Found author {author}, created: {created}") + + book, created = Book.objects.get_or_create( + koreader_md5=book_row[KoReaderBookColumn.MD5.value] + ) + + if created: + book.title = book_row[KoReaderBookColumn.TITLE.value] + book.pages = book_row[KoReaderBookColumn.PAGES.value] + book.koreader_id = int(book_row[KoReaderBookColumn.ID.value]) + book.koreader_authors = book_row[KoReaderBookColumn.AUTHORS.value] + book.run_time_ticks = int(book_row[KoReaderBookColumn.PAGES.value]) + book.save( + update_fields=[ + "title", + "pages", + "koreader_id", + "koreader_authors", + ] + ) + book.fix_metadata() + if author_list: + book.authors.add(*[a.id for a in author_list]) + + playback_position = int( + book_row[KoReaderBookColumn.TOTAL_READ_TIME.value] + ) + playback_position_ticks = playback_position * 1000 + pages_read = int(book_row[KoReaderBookColumn.TOTAL_READ_PAGES.value]) + timestamp = datetime.utcfromtimestamp( + book_row[KoReaderBookColumn.LAST_OPEN.value] + ).replace(tzinfo=pytz.utc) + + new_scrobble = Scrobble( + book_id=book.id, + user_id=user_id, + source="KOReader", + timestamp=timestamp, + playback_position_ticks=playback_position_ticks, + playback_position=playback_position, + played_to_completion=True, + in_progress=False, + book_pages_read=pages_read, + ) + + existing = Scrobble.objects.filter( + timestamp=timestamp, book=book + ).first() + if existing: + logger.debug(f"Skipping existing scrobble {new_scrobble}") + continue + + logger.debug(f"Queued scrobble {new_scrobble} for creation") + new_scrobbles.append(new_scrobble) + + # Be sure to close the connection + con.close() + + created = Scrobble.objects.bulk_create(new_scrobbles) + logger.info( + f"Created {len(created)} scrobbles", + extra={'created_scrobbles': created}, + ) + return created diff --git a/vrobbler/apps/scrobbles/migrations/0020_alter_audioscrobblertsvimport_options_and_more.py b/vrobbler/apps/scrobbles/migrations/0020_alter_audioscrobblertsvimport_options_and_more.py new file mode 100644 index 0000000..38f6423 --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0020_alter_audioscrobblertsvimport_options_and_more.py @@ -0,0 +1,97 @@ +# Generated by Django 4.1.5 on 2023-02-19 03:53 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import django_extensions.db.fields +import scrobbles.models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ( + 'scrobbles', + '0019_rename_processed_on_lastfmimport_processed_finished_and_more', + ), + ] + + operations = [ + migrations.AlterModelOptions( + name='audioscrobblertsvimport', + options={}, + ), + migrations.AlterModelOptions( + name='lastfmimport', + options={}, + ), + migrations.RenameField( + model_name='audioscrobblertsvimport', + old_name='processed_on', + new_name='processed_finished', + ), + migrations.AddField( + model_name='audioscrobblertsvimport', + name='processing_started', + field=models.DateTimeField(blank=True, null=True), + ), + migrations.CreateModel( + name='KoReaderImport', + fields=[ + ( + 'id', + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + ), + ), + ( + 'created', + django_extensions.db.fields.CreationDateTimeField( + auto_now_add=True, verbose_name='created' + ), + ), + ( + 'modified', + django_extensions.db.fields.ModificationDateTimeField( + auto_now=True, verbose_name='modified' + ), + ), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False)), + ( + 'processing_started', + models.DateTimeField(blank=True, null=True), + ), + ( + 'processed_finished', + models.DateTimeField(blank=True, null=True), + ), + ('process_log', models.TextField(blank=True, null=True)), + ('process_count', models.IntegerField(blank=True, null=True)), + ( + 'sqlite_file', + models.FileField( + blank=True, + null=True, + upload_to=scrobbles.models.KoReaderImport.get_path, + ), + ), + ( + 'user', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/vrobbler/apps/scrobbles/migrations/0021_scrobble_book.py b/vrobbler/apps/scrobbles/migrations/0021_scrobble_book.py new file mode 100644 index 0000000..c614e7e --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0021_scrobble_book.py @@ -0,0 +1,25 @@ +# Generated by Django 4.1.5 on 2023-02-19 20:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('books', '0001_initial'), + ('scrobbles', '0020_alter_audioscrobblertsvimport_options_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='scrobble', + name='book', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.DO_NOTHING, + to='books.book', + ), + ), + ] diff --git a/vrobbler/apps/scrobbles/migrations/0022_scrobble_book_pages_read.py b/vrobbler/apps/scrobbles/migrations/0022_scrobble_book_pages_read.py new file mode 100644 index 0000000..2e28082 --- /dev/null +++ b/vrobbler/apps/scrobbles/migrations/0022_scrobble_book_pages_read.py @@ -0,0 +1,18 @@ +# Generated by Django 4.1.5 on 2023-02-20 00:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scrobbles', '0021_scrobble_book'), + ] + + operations = [ + migrations.AddField( + model_name='scrobble', + name='book_pages_read', + field=models.IntegerField(blank=True, null=True), + ), + ] diff --git a/vrobbler/apps/scrobbles/models.py b/vrobbler/apps/scrobbles/models.py index 2c10147..c5fd015 100644 --- a/vrobbler/apps/scrobbles/models.py +++ b/vrobbler/apps/scrobbles/models.py @@ -7,6 +7,7 @@ from django.db import models from django.utils import timezone from django_extensions.db.models import TimeStampedModel from music.models import Artist, Track +from books.models import Book from podcasts.models import Episode from profiles.utils import now_user_timezone from scrobbles.lastfm import LastFM @@ -19,62 +20,7 @@ User = get_user_model() BNULL = {"blank": True, "null": True} -class AudioScrobblerTSVImport(TimeStampedModel): - def get_path(instance, filename): - extension = filename.split('.')[-1] - uuid = instance.uuid - return f'audioscrobbler-uploads/{uuid}.{extension}' - - user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL) - uuid = models.UUIDField(editable=False, default=uuid4) - tsv_file = models.FileField(upload_to=get_path, **BNULL) - processed_on = models.DateTimeField(**BNULL) - process_log = models.TextField(**BNULL) - process_count = models.IntegerField(**BNULL) - - def __str__(self): - if self.tsv_file: - return f"Audioscrobbler TSV upload: {self.tsv_file.path}" - return f"Audioscrobbler TSV upload {self.id}" - - def process(self, force=False): - from scrobbles.tsv import process_audioscrobbler_tsv_file - - if self.processed_on and not force: - logger.info(f"{self} already processed on {self.processed_on}") - return - - tz = None - if self.user: - tz = self.user.profile.tzinfo - scrobbles = process_audioscrobbler_tsv_file( - self.tsv_file.path, self.user.id, user_tz=tz - ) - self.process_log = "" - if scrobbles: - for count, scrobble in enumerate(scrobbles): - scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}" - log_line = f"{scrobble_str}" - if count > 0: - log_line = "\n" + log_line - self.process_log += log_line - self.process_count = len(scrobbles) - else: - self.process_log = f"Created no new scrobbles" - self.process_count = 0 - - self.processed_on = timezone.now() - self.save( - update_fields=['processed_on', 'process_count', 'process_log'] - ) - - def undo(self, dryrun=True): - from scrobbles.tsv import undo_audioscrobbler_tsv_import - - undo_audioscrobbler_tsv_import(self.process_log, dryrun) - - -class LastFmImport(TimeStampedModel): +class BaseFileImportMixin(TimeStampedModel): user = models.ForeignKey(User, on_delete=models.DO_NOTHING, **BNULL) uuid = models.UUIDField(editable=False, default=uuid4) processing_started = models.DateTimeField(**BNULL) @@ -82,8 +28,135 @@ class LastFmImport(TimeStampedModel): process_log = models.TextField(**BNULL) process_count = models.IntegerField(**BNULL) + class Meta: + abstract = True + def __str__(self): - return f"LastFM Import: {self.uuid}" + return f"Scrobble import {self.id}" + + def process(self, force=False): + logger.warning("Process not implemented") + + def undo(self, dryrun=False): + """Accepts the log from a scrobble import and removes the scrobbles""" + from scrobbles.models import Scrobble + + if not self.process_log: + logger.warning("No lines in process log found to undo") + return + + for line in self.process_log.split('\n'): + scrobble_id = line.split("\t")[0] + scrobble = Scrobble.objects.filter(id=scrobble_id).first() + if not scrobble: + logger.warning( + f"Could not find scrobble {scrobble_id} to undo" + ) + continue + logger.info(f"Removing scrobble {scrobble_id}") + if not dryrun: + scrobble.delete() + self.processed_finished = None + self.processing_started = None + self.process_count = None + self.process_log = "" + self.save( + update_fields=[ + "processed_finished", + "processing_started", + "process_log", + "process_count", + ] + ) + + def mark_started(self): + self.processing_started = timezone.now() + self.save(update_fields=["processing_started"]) + + def mark_finished(self): + self.processed_finished = timezone.now() + self.save(update_fields=['processed_finished']) + + def record_log(self, scrobbles): + self.process_log = "" + if not scrobbles: + self.process_count = 0 + self.save(update_fields=["process_log" "process_count"]) + return + + for count, scrobble in enumerate(scrobbles): + scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.media_obj.title}" + log_line = f"{scrobble_str}" + if count > 0: + log_line = "\n" + log_line + self.process_log += log_line + self.process_count = len(scrobbles) + self.save(update_fields=["process_log", "process_count"]) + + +class KoReaderImport(BaseFileImportMixin): + class Meta: + verbose_name = "KOReader Import" + + def get_path(instance, filename): + extension = filename.split('.')[-1] + uuid = instance.uuid + return f'koreader-uploads/{uuid}.{extension}' + + sqlite_file = models.FileField(upload_to=get_path, **BNULL) + + def process(self, force=False): + from scrobbles.koreader import process_koreader_sqlite_file + + if self.processed_finished and not force: + logger.info( + f"{self} already processed on {self.processed_finished}" + ) + return + + self.mark_started() + scrobbles = process_koreader_sqlite_file( + self.sqlite_file.path, self.user.id + ) + self.record_log(scrobbles) + self.mark_finished() + + +class AudioScrobblerTSVImport(BaseFileImportMixin): + class Meta: + verbose_name = "AudioScrobbler TSV Import" + + def get_path(instance, filename): + extension = filename.split('.')[-1] + uuid = instance.uuid + return f'audioscrobbler-uploads/{uuid}.{extension}' + + tsv_file = models.FileField(upload_to=get_path, **BNULL) + + def process(self, force=False): + from scrobbles.tsv import process_audioscrobbler_tsv_file + + if self.processed_finished and not force: + logger.info( + f"{self} already processed on {self.processed_finished}" + ) + return + + self.mark_started() + + tz = None + if self.user: + tz = self.user.profile.tzinfo + scrobbles = process_audioscrobbler_tsv_file( + self.tsv_file.path, self.user.id, user_tz=tz + ) + self.record_log(scrobbles) + self.mark_finished() + + +class LastFmImport(BaseFileImportMixin): + class Meta: + verbose_name = "Last.FM Import" def process(self, import_all=False): """Import scrobbles found on LastFM""" @@ -111,36 +184,12 @@ class LastFmImport(TimeStampedModel): if last_import: last_processed = last_import.processed_finished - self.processing_started = timezone.now() - self.save(update_fields=['processing_started']) + self.mark_started() scrobbles = lastfm.import_from_lastfm(last_processed) - self.process_log = "" - if scrobbles: - for count, scrobble in enumerate(scrobbles): - scrobble_str = f"{scrobble.id}\t{scrobble.timestamp}\t{scrobble.track.title}" - log_line = f"{scrobble_str}" - if count > 0: - log_line = "\n" + log_line - self.process_log += log_line - self.process_count = len(scrobbles) - else: - self.process_count = 0 - self.processed_finished = timezone.now() - self.save( - update_fields=[ - 'processed_finished', - 'process_count', - 'process_log', - ] - ) - - def undo(self, dryrun=False): - """Undo import of scrobbles from LastFM""" - LastFM.undo_lastfm_import(self.process_log, dryrun) - self.processed_finished = None - self.save(update_fields=['processed_finished']) + self.record_log(scrobbles) + self.mark_finished() class ChartRecord(TimeStampedModel): @@ -231,19 +280,29 @@ class Scrobble(TimeStampedModel): sport_event = models.ForeignKey( SportEvent, on_delete=models.DO_NOTHING, **BNULL ) + book = models.ForeignKey(Book, on_delete=models.DO_NOTHING, **BNULL) user = models.ForeignKey( User, blank=True, null=True, on_delete=models.DO_NOTHING ) + + # Time keeping timestamp = models.DateTimeField(**BNULL) playback_position_ticks = models.PositiveBigIntegerField(**BNULL) playback_position = models.CharField(max_length=8, **BNULL) + + # Status indicators is_paused = models.BooleanField(default=False) played_to_completion = models.BooleanField(default=False) + in_progress = models.BooleanField(default=True) + + # Metadata source = models.CharField(max_length=255, **BNULL) source_id = models.TextField(**BNULL) - in_progress = models.BooleanField(default=True) scrobble_log = models.TextField(**BNULL) + # Fields for keeping track of reads between scrobbles + book_pages_read = models.IntegerField(**BNULL) + def save(self, *args, **kwargs): if not self.uuid: self.uuid = uuid4() @@ -272,7 +331,10 @@ class Scrobble(TimeStampedModel): @property def percent_played(self) -> int: - if not self.media_obj.run_time_ticks: + if not self.media_obj: + return 0 + + if self.media_obj and not self.media_obj.run_time_ticks: return 100 if not self.playback_position_ticks and self.played_to_completion: @@ -309,6 +371,8 @@ class Scrobble(TimeStampedModel): media_obj = self.podcast_episode if self.sport_event: media_obj = self.sport_event + if self.book: + media_obj = self.book return media_obj def __str__(self): @@ -332,6 +396,9 @@ class Scrobble(TimeStampedModel): if media.__class__.__name__ == 'SportEvent': media_query = models.Q(sport_event=media) scrobble_data['sport_event_id'] = media.id + if media.__class__.__name__ == 'Book': + media_query = models.Q(book=media) + scrobble_data['book_id'] = media.id scrobble = ( cls.objects.filter( diff --git a/vrobbler/apps/scrobbles/tasks.py b/vrobbler/apps/scrobbles/tasks.py index aebeb6f..aa30184 100644 --- a/vrobbler/apps/scrobbles/tasks.py +++ b/vrobbler/apps/scrobbles/tasks.py @@ -1,7 +1,11 @@ import logging from celery import shared_task -from scrobbles.models import AudioScrobblerTSVImport, LastFmImport +from scrobbles.models import ( + AudioScrobblerTSVImport, + KoReaderImport, + LastFmImport, +) logger = logging.getLogger(__name__) @@ -22,3 +26,12 @@ def process_tsv_import(import_id): logger.warn(f"AudioScrobblerTSVImport not found with id {import_id}") tsv_import.process() + + +@shared_task +def process_koreader_import(import_id): + koreader_import = KoReaderImport.objects.filter(id=import_id).first() + if not koreader_import: + logger.warn(f"KOReaderImport not found with id {import_id}") + + koreader_import.process() diff --git a/vrobbler/apps/scrobbles/tsv.py b/vrobbler/apps/scrobbles/tsv.py index 7a66b62..f105fc7 100644 --- a/vrobbler/apps/scrobbles/tsv.py +++ b/vrobbler/apps/scrobbles/tsv.py @@ -78,20 +78,3 @@ def process_audioscrobbler_tsv_file(file_path, user_id, user_tz=None): extra={'created_scrobbles': created}, ) return created - - -def undo_audioscrobbler_tsv_import(process_log, dryrun=False): - """Accepts the log from a TSV import and removes the scrobbles""" - if not process_log: - logger.warning("No lines in process log found to undo") - return - - for line in process_log.split('\n'): - scrobble_id = line.split("\t")[0] - scrobble = Scrobble.objects.filter(id=scrobble_id).first() - if not scrobble: - logger.warning(f"Could not find scrobble {scrobble_id} to undo") - continue - logger.info(f"Removing scrobble {scrobble_id}") - if not dryrun: - scrobble.delete() diff --git a/vrobbler/settings.py b/vrobbler/settings.py index 6f11aa9..50cfd07 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -97,6 +97,7 @@ INSTALLED_APPS = [ "music", "podcasts", "sports", + "books", "mathfilters", "rest_framework", "allauth",