Add proper author lookups and fix bad OL fixes

vrobbler/apps/books/koreader.py

@@ -12,6 +12,7 @@ from books.models import Author, Book, Page
 from pylast import httpx, tempfile
 from scrobbles.models import Scrobble
 from stream_sqlite import stream_sqlite
+from vrobbler.apps.books.openlibrary import get_author_openlibrary_id
 
 logger = logging.getLogger(__name__)
 
@@ -53,39 +54,55 @@ def get_book_map_from_sqlite(rows: Iterable) -> dict:
     book_id_map = {}
 
     for book_row in rows:
-        authors = book_row[KoReaderBookColumn.AUTHORS.value].split("\n")
+        book = Book.objects.filter(
-        author_list = []
+            koreader_md5=book_row[KoReaderBookColumn.MD5.value]
-        for author_str in authors:
+        ).first()
-            logger.debug(f"Looking up author {author_str}")
+
-            if author_str == "N/A":
+        if not book:
-                continue
+            book, created = Book.objects.get_or_create(
+                title=book_row[KoReaderBookColumn.TITLE.value]
+            )
 
-            author, created = Author.objects.get_or_create(name=author_str)
             if created:
-                author.fix_metadata()
+                total_pages = book_row[KoReaderBookColumn.PAGES.value]
-            author_list.append(author)
+                run_time = total_pages * book.AVG_PAGE_READING_SECONDS
-            logger.debug(f"Found author {author}, created: {created}")
+                ko_authors = book_row[
+                    KoReaderBookColumn.AUTHORS.value
+                ].replace("\n", ", ")
+                book_dict = {
+                    "title": book_row[KoReaderBookColumn.TITLE.value],
+                    "pages": total_pages,
+                    "koreader_md5": book_row[KoReaderBookColumn.MD5.value],
+                    "koreader_id": int(book_row[KoReaderBookColumn.ID.value]),
+                    "koreader_authors": ko_authors,
+                    "run_time_seconds": run_time,
+                }
+                Book.objects.filter(pk=book.id).update(**book_dict)
 
-        book, created = Book.objects.get_or_create(
+                # Add authors
-            title=book_row[KoReaderBookColumn.TITLE.value]
+                authors = book_row[KoReaderBookColumn.AUTHORS.value].split(
-        )
+                    "\n"
+                )
+                author_list = []
+                for author_str in authors:
+                    logger.debug(f"Looking up author {author_str}")
+                    if author_str == "N/A":
+                        continue
 
-        if created:
+                    author, created = Author.objects.get_or_create(
-            total_pages = book_row[KoReaderBookColumn.PAGES.value]
+                        name=author_str
-            run_time = total_pages * book.AVG_PAGE_READING_SECONDS
+                    )
-            book_dict = {
+                    if created:
-                "title": book_row[KoReaderBookColumn.TITLE.value],
+                        author.openlibrary_id = get_author_openlibrary_id(
-                "pages": total_pages,
+                            author_str
-                "koreader_md5": book_row[KoReaderBookColumn.MD5.value],
+                        )
-                "koreader_id": int(book_row[KoReaderBookColumn.ID.value]),
+                        author.save(update_fields=["openlibrary_id"])
-                "koreader_authors": book_row[KoReaderBookColumn.AUTHORS.value],
+                        author.fix_metadata()
-                "run_time_seconds": run_time,
+                        logger.debug(f"Created author {author}")
-            }
+                    book.authors.add(author)
-            Book.objects.filter(pk=book.id).update(**book_dict)
-            book.fix_metadata()
 
-            if author_list:
+                # This will try to fix metadata by looking it up on OL
-                book.authors.add(*[a.id for a in author_list])
+                book.fix_metadata()
 
         playback_position_seconds = 0
         if book_row[KoReaderBookColumn.TOTAL_READ_TIME.value]:
@@ -101,6 +118,7 @@ def get_book_map_from_sqlite(rows: Iterable) -> dict:
         timestamp = datetime.utcfromtimestamp(
             book_row[KoReaderBookColumn.LAST_OPEN.value]
         ).replace(tzinfo=pytz.utc)
+        book.refresh_from_db()
         book_id_map[book.koreader_id] = book.id
 
     return book_id_map
@@ -130,6 +148,7 @@ def build_scrobbles_from_pages(
             ]
             page.save(update_fields=["start_time", "duration_seconds"])
             page.refresh_from_db()
+
         if page.is_scrobblable:
             # Page number is a placeholder, we'll re-preocess this after creation
             logger.debug(
@@ -155,11 +174,12 @@ def enrich_koreader_scrobbles(scrobbles: list) -> None:
 
     for scrobble in scrobbles:
         if scrobble.next:
+            # Set pages read to the starting page of the next scrobble minus one, if it exists
             scrobble.book_pages_read = scrobble.next.book_pages_read - 1
             scrobble.save(update_fields=["book_pages_read"])
         else:
+            # Set pages read to the last page we have
             scrobble.book_pages_read = scrobble.book.page_set.last().number
-            scrobble.long_play_complete =
 
         scrobble.save(update_fields=["book_pages_read", "long_play_complete"])
 

vrobbler/apps/books/models.py

@@ -98,7 +98,10 @@ class Book(LongPlayScrobblableMixin):
 
     def fix_metadata(self, force_update=False):
         if not self.openlibrary_id or force_update:
-            book_dict = lookup_book_from_openlibrary(self.title, self.author)
+            author_name = ""
+            if self.author:
+                author_name = self.author.name
+            book_dict = lookup_book_from_openlibrary(self.title, author_name)
             if not book_dict:
                 logger.warn(f"Book not found in OL {self.title}")
                 return
@@ -115,7 +118,6 @@ class Book(LongPlayScrobblableMixin):
                 logger.warn(
                     f"OL and KoReader disagree on this book title {self.title} != {ol_title}"
                 )
-                return
 
             Book.objects.filter(pk=self.id).update(**book_dict)
             self.refresh_from_db()

vrobbler/apps/books/openlibrary.py

@@ -1,14 +1,15 @@
 import json
 import logging
+import re
 import urllib
 
 import requests
 
 logger = logging.getLogger(__name__)
 
-SEARCH_URL = "https://openlibrary.org/search.json?title={title}"
 ISBN_URL = "https://openlibrary.org/isbn/{isbn}.json"
-SEARCH_URL = "https://openlibrary.org/search.json?title={title}"
+SEARCH_URL = "https://openlibrary.org/search.json?q={query}&sort=editions&mode=everything"
+AUTHOR_SEARCH_URL = "https://openlibrary.org/search/authors.json?q={query}"
 COVER_URL = "https://covers.openlibrary.org/b/olid/{id}-L.jpg"
 AUTHOR_URL = "https://openlibrary.org/authors/{id}.json"
 AUTHOR_IMAGE_URL = "https://covers.openlibrary.org/a/olid/{id}-L.jpg"
@@ -21,6 +22,24 @@ def get_first(key: str, result: dict) -> str:
     return obj
 
 
+def get_author_openlibrary_id(name: str) -> str:
+    search_url = AUTHOR_SEARCH_URL.format(query=name)
+    response = requests.get(search_url)
+
+    if response.status_code != 200:
+        logger.warn(f"Bad response from OL: {response.status_code}")
+        return ""
+
+    results = json.loads(response.content)
+
+    if not results:
+        logger.warn(f"No author results found from search for {name}")
+        return ""
+
+    result = results.get("docs", [])
+    return result[0].get("key")
+
+
 def lookup_author_from_openlibrary(olid: str) -> dict:
     author_url = AUTHOR_URL.format(id=olid)
     response = requests.get(author_url)
@@ -58,7 +77,14 @@ def lookup_author_from_openlibrary(olid: str) -> dict:
 
 def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
     title_quoted = urllib.parse.quote(title)
-    search_url = SEARCH_URL.format(title=title_quoted)
+    author_quoted = ""
+    if author:
+        # Strip middle initials, OpenLibrary often fails with these
+        author = re.sub(" [A-Z]. ", " ", author)
+        author_quoted = urllib.parse.quote(author)
+    query = f"{title_quoted} {author_quoted}"
+
+    search_url = SEARCH_URL.format(query=query)
     response = requests.get(search_url)
 
     if response.status_code != 200:
@@ -71,7 +97,17 @@ def lookup_book_from_openlibrary(title: str, author: str = None) -> dict:
         logger.warn(f"No results found from OL for {title}")
         return {}
 
-    top = results.get("docs")[0]
+    top = None
+    for result in results.get("docs"):
+        # These Summary things suck and ruin our one-shot search
+        if "Summary of" not in result.get("title"):
+            top = result
+            break
+
+    if not top:
+        logger.warn(f"No book found for query {query}")
+        return {}
+
     ol_id = top.get("cover_edition_key")
     ol_author_id = get_first("author_key", top)
     first_sentence = ""