[books] Add goodreads exports

PROJECT.org

@@ -524,23 +524,6 @@ AllTrails is the best source, though having TrailForks is nice to.
 Would be nice to have some loose connection to the actual event in my Garmin
 profile.
 
-** TODO [#B] Fix how we show notes and descriptions from scrobbles to users :metadata:notes:tasks:
-:PROPERTIES:
-:ID:       adf4c513-a417-4ec9-8831-f01ffcf63276
-:END:
-*** Description
-
-Currently the display of notes leaves something to be desired. The biggest issue
-is that they don't look good on mobile and are probably trying to be too cute.
-Rather than post-it note style, we should just put notes in a list under the
-description, above the Edit Log toggle, with timestamps for when they were
-added.
-
-They should also probably support markdown formatting and that should be
-displayed in the template.
-
-** TODO [#B] Add CSV endpoint for book scrobbles that LibraryThing can ingest :books:feature:export:
-https://app.todoist.com/app/task/add-a-csv-endpoint-for-users-book-reads-that-library-thing-can-ingest-6X7QPMRp265xMXqg#comment-6X7QrXq6gJjMP4hg
 ** TODO [#B] Make IMAP and WebDAV configurable :webdav:feature:imap:importers:
 :PROPERTIES:
 :ID:       b1426d92-2feb-4d15-9738-d5b7b0594f96
@@ -611,6 +594,32 @@ The Edit log form should have from top to bottom:
   - People (which should be similar to the Bird widget on BirdLocation and allow setting per user score, win true/false, rank, new true/false, seat_ordrer)
   - Expansion ids (which should a multi-select widget of expansions for this game)
   - Location (which should be a drop down of BoardGameLocations for this user)
+** DONE [#B] Add CSV endpoint for book scrobbles that LibraryThing can ingest :books:feature:export:
+:PROPERTIES:
+:ID:       fd8a2778-ff46-394b-9868-edb9a48d0ce6
+:END:
+
+
+*** Description
+We should provide an "Export for LibraryThings" link on the Books list page
+which creates a CSV file of all books and tags for "in-progress" or "finished"
+depending on whether the book has been marked as finished.
+
+[[https://www.librarything.com/LibraryThingSample.csv][Sample CSV file]]
+
+*** Implementation
+
+Added a `books:librarything_export` endpoint (a `LoginRequiredMixin` view) that
+streams a CSV matching the [[https://www.librarything.com/LibraryThingSample.csv][LibraryThing sample]]
+column layout (TITLE, AUTHOR (last, first), DATE, ISBN, PUBLICATION INFO, TAGS,
+RATING, REVIEW, DATE READ, PAGE COUNT, CALL NUMBER). The `TAGS` column is set to
+"in-progress" or "finished" based on the user's last scrobble's
+`long_play_complete` flag, `DATE READ` reflects that scrobble's timestamp when
+finished, author names are reformatted to "last, first", and the export is
+limited to books the user has scrobbled. The CSV logic lives in
+`books.utils.librarything_csv_rows()` with a `format_author_last_first()`
+helper; a link on the Books list page (`book_list.html`) triggers the download.
+
 ** DONE [#A] Fix recursive bug in backup script :backups:bug:
 :PROPERTIES:
 :ID:       e986d35e-fc23-5d31-9d2a-0a95da4748a2

tests/books_tests/test_goodreads_export.py

@@ -0,0 +1,133 @@
+import csv
+from datetime import datetime
+from io import StringIO
+
+import pytest
+from books.models import Author, Book
+from books.utils import GOODREADS_HEADERS
+from django.contrib.auth import get_user_model
+from django.urls import reverse
+from django.utils import timezone
+from scrobbles.models import Scrobble
+
+User = get_user_model()
+
+
+@pytest.fixture
+def user():
+    return User.objects.create_user(
+        username="reader", email="reader@example.com", password="testpass"
+    )
+
+
+@pytest.fixture
+def book(user):
+    primary = Author.objects.create(name="Nicholas A. Basbanes")
+    secondary = Author.objects.create(name="Jane Q. Doe")
+    book = Book.objects.create(
+        title="A Gentle Madness",
+        isbn_13="9780805061765",
+        isbn_10="0805061762",
+        pages=500,
+        publisher="Holt Paperbacks",
+        first_publish_year=1999,
+    )
+    book.authors.add(primary, secondary)
+    book.tags.add("history", "books")
+    return book
+
+
+def test_goodreads_export_requires_login(client):
+    url = reverse("books:goodreads_export")
+    response = client.get(url)
+    assert response.status_code == 302
+
+
+@pytest.mark.django_db
+def test_goodreads_export_empty(client, user):
+    client.force_login(user)
+    url = reverse("books:goodreads_export")
+    response = client.get(url)
+    assert response.status_code == 200
+    assert response["Content-Type"] == "text/csv"
+    rows = list(csv.reader(StringIO(response.content.decode())))
+    assert rows[0] == GOODREADS_HEADERS
+    assert len(rows) == 1
+
+
+@pytest.mark.django_db
+def test_goodreads_export_marks_in_progress(client, user, book):
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 3, 1)),
+        long_play_complete=False,
+    )
+
+    client.force_login(user)
+    url = reverse("books:goodreads_export")
+    response = client.get(url)
+    rows = list(csv.DictReader(StringIO(response.content.decode())))
+    assert len(rows) == 1
+    assert rows[0]["Title"] == "A Gentle Madness"
+    assert rows[0]["Author"] == "Nicholas A. Basbanes"
+    assert rows[0]["Author l-f"] == "Basbanes, Nicholas A."
+    assert rows[0]["Additional Authors"] == "Jane Q. Doe"
+    assert rows[0]["ISBN"] == "0805061762"
+    assert rows[0]["ISBN13"] == "9780805061765"
+    assert rows[0]["Number of Pages"] == "500"
+    assert rows[0]["Year Published"] == "1999"
+    assert rows[0]["Exclusive Shelf"] == "currently-reading"
+    assert rows[0]["Date Read"] == ""
+    assert rows[0]["Date Added"] == "2023/03/01"
+    assert rows[0]["Bookshelves"] == "books, history"
+    assert rows[0]["Read Count"] == "1"
+    assert rows[0]["Owned Copies"] == "0"
+
+
+@pytest.mark.django_db
+def test_goodreads_export_marks_finished(client, user, book):
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 5, 5)),
+        long_play_complete=True,
+    )
+
+    client.force_login(user)
+    url = reverse("books:goodreads_export")
+    response = client.get(url)
+    rows = list(csv.DictReader(StringIO(response.content.decode())))
+    assert len(rows) == 1
+    assert rows[0]["Exclusive Shelf"] == "read"
+    assert rows[0]["Date Read"] == "2023/05/05"
+
+
+@pytest.mark.django_db
+def test_goodreads_export_date_added_is_first_scrobble(client, user, book):
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 3, 1)),
+        long_play_complete=False,
+    )
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 5, 5)),
+        long_play_complete=True,
+    )
+
+    client.force_login(user)
+    url = reverse("books:goodreads_export")
+    response = client.get(url)
+    rows = list(csv.DictReader(StringIO(response.content.decode())))
+    assert len(rows) == 1
+    assert rows[0]["Exclusive Shelf"] == "read"
+    assert rows[0]["Date Added"] == "2023/03/01"
+    assert rows[0]["Date Read"] == "2023/05/05"
+    assert rows[0]["Read Count"] == "2"

tests/books_tests/test_librarything_export.py

@@ -0,0 +1,135 @@
+import csv
+from datetime import datetime
+from io import StringIO
+
+import pytest
+from books.models import Author, Book
+from django.contrib.auth import get_user_model
+from django.urls import reverse
+from django.utils import timezone
+from scrobbles.models import Scrobble
+
+User = get_user_model()
+
+
+@pytest.fixture
+def user():
+    return User.objects.create_user(
+        username="reader", email="reader@example.com", password="testpass"
+    )
+
+
+@pytest.fixture
+def book(user):
+    author = Author.objects.create(name="Nicholas A. Basbanes")
+    book = Book.objects.create(
+        title="A Gentle Madness",
+        isbn_13="0805061762",
+        pages=500,
+        publisher="Holt Paperbacks",
+        first_publish_year=1999,
+    )
+    book.authors.add(author)
+    return book
+
+
+def test_librarything_export_requires_login(client):
+    url = reverse("books:librarything_export")
+    response = client.get(url)
+    assert response.status_code == 302
+
+
+@pytest.mark.django_db
+def test_librarything_export_empty(client, user):
+    client.force_login(user)
+    url = reverse("books:librarything_export")
+    response = client.get(url)
+    assert response.status_code == 200
+    assert response["Content-Type"] == "text/csv"
+    rows = list(csv.reader(StringIO(response.content.decode())))
+    assert rows[0] == [
+        "TITLE",
+        "AUTHOR (last, first)",
+        "DATE",
+        "ISBN",
+        "PUBLICATION INFO",
+        "TAGS",
+        "RATING",
+        "REVIEW",
+        "DATE READ",
+        "ENTRY DATE",
+        "PAGE COUNT",
+        "CALL NUMBER",
+    ]
+    assert len(rows) == 1
+
+
+@pytest.mark.django_db
+def test_librarything_export_marks_in_progress(client, user, book):
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 3, 1)),
+        long_play_complete=False,
+    )
+
+    client.force_login(user)
+    url = reverse("books:librarything_export")
+    response = client.get(url)
+    rows = list(csv.DictReader(StringIO(response.content.decode())))
+    assert len(rows) == 1
+    assert rows[0]["TITLE"] == "A Gentle Madness"
+    assert rows[0]["AUTHOR (last, first)"] == "Basbanes, Nicholas A."
+    assert rows[0]["ISBN"] == "0805061762"
+    assert rows[0]["PAGE COUNT"] == "500"
+    assert rows[0]["TAGS"] == "in-progress"
+    assert rows[0]["DATE READ"] == ""
+    assert rows[0]["ENTRY DATE"] == "2023-03-01"
+
+
+@pytest.mark.django_db
+def test_librarything_export_marks_finished(client, user, book):
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 5, 5)),
+        long_play_complete=True,
+    )
+
+    client.force_login(user)
+    url = reverse("books:librarything_export")
+    response = client.get(url)
+    rows = list(csv.DictReader(StringIO(response.content.decode())))
+    assert len(rows) == 1
+    assert rows[0]["TAGS"] == "finished"
+    assert rows[0]["DATE READ"] == "2023-05-05"
+    assert rows[0]["ENTRY DATE"] == "2023-05-05"
+
+
+@pytest.mark.django_db
+def test_librarything_export_entry_date_is_first_scrobble(client, user, book):
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 3, 1)),
+        long_play_complete=False,
+    )
+    Scrobble.objects.create(
+        book=book,
+        media_type=Scrobble.MediaType.BOOK,
+        user=user,
+        timestamp=timezone.make_aware(datetime(2023, 5, 5)),
+        long_play_complete=True,
+    )
+
+    client.force_login(user)
+    url = reverse("books:librarything_export")
+    response = client.get(url)
+    rows = list(csv.DictReader(StringIO(response.content.decode())))
+    assert len(rows) == 1
+    assert rows[0]["TAGS"] == "finished"
+    assert rows[0]["ENTRY DATE"] == "2023-03-01"
+    assert rows[0]["DATE READ"] == "2023-05-05"

vrobbler/apps/books/urls.py

@@ -1,5 +1,5 @@
-from django.urls import path
 from books import views
+from django.urls import path
 
 app_name = "books"
 
@@ -27,4 +27,14 @@ urlpatterns = [
         views.PaperUploadPdfView.as_view(),
         name="paper_upload_pdf",
     ),
+    path(
+        "books/export/librarything/",
+        views.LibraryThingExportView.as_view(),
+        name="librarything_export",
+    ),
+    path(
+        "books/export/goodreads/",
+        views.GoodreadsExportView.as_view(),
+        name="goodreads_export",
+    ),
 ]

vrobbler/apps/books/utils.py

@@ -3,6 +3,186 @@ from urllib.parse import urlparse, urlunparse
 
 from titlecase import titlecase
 
+LIBRARYTHING_HEADERS = [
+    "TITLE",
+    "AUTHOR (last, first)",
+    "DATE",
+    "ISBN",
+    "PUBLICATION INFO",
+    "TAGS",
+    "RATING",
+    "REVIEW",
+    "DATE READ",
+    "ENTRY DATE",
+    "PAGE COUNT",
+    "CALL NUMBER",
+]
+
+GOODREADS_HEADERS = [
+    "Book Id",
+    "Title",
+    "Author",
+    "Author l-f",
+    "Additional Authors",
+    "ISBN",
+    "ISBN13",
+    "My Rating",
+    "Average Rating",
+    "Publisher",
+    "Binding",
+    "Number of Pages",
+    "Year Published",
+    "Original Publication Year",
+    "Date Read",
+    "Date Added",
+    "Bookshelves",
+    "Bookshelves with positions",
+    "Exclusive Shelf",
+    "My Review",
+    "Spoiler",
+    "Private Notes",
+    "Read Count",
+    "Recommended For",
+    "Recommended By",
+    "Owned Copies",
+    "Original Purchase Date",
+    "Original Purchase Location",
+    "Condition",
+    "Condition Description",
+    "BCID",
+]
+
+
+def format_author_last_first(name: str) -> str:
+    parts = name.strip().split()
+    if len(parts) < 2:
+        return name
+    return f"{parts[-1]}, {' '.join(parts[:-1])}"
+
+
+def librarything_csv_rows(user) -> list[list]:
+    from books.models import Book
+
+    books = Book.objects.filter(scrobble__user=user).distinct().order_by("title")
+
+    rows = []
+    for book in books:
+        user_scrobbles = book.scrobble_set.filter(user=user)
+        last_scrobble = user_scrobbles.order_by("-timestamp").first()
+        first_scrobble = user_scrobbles.order_by("timestamp").first()
+        finished = bool(last_scrobble and last_scrobble.long_play_complete)
+        status = "finished" if finished else "in-progress"
+
+        authors = ", ".join(
+            format_author_last_first(a.name) for a in book.authors.all()
+        )
+        publish_year = (
+            book.publish_date.year
+            if book.publish_date
+            else (book.first_publish_year or "")
+        )
+        isbn = book.isbn_13 or book.isbn_10 or ""
+        pub_info = ""
+        if book.publisher:
+            pub_info = book.publisher
+        if publish_year:
+            pub_info = f"{pub_info} ({publish_year})".strip()
+        if book.pages:
+            pub_info = f"{pub_info}, {book.pages} pages"
+        date_read = ""
+        if finished and last_scrobble.timestamp:
+            date_read = last_scrobble.timestamp.strftime("%Y-%m-%d")
+        entry_date = ""
+        if first_scrobble and first_scrobble.timestamp:
+            entry_date = first_scrobble.timestamp.strftime("%Y-%m-%d")
+
+        rows.append(
+            [
+                book.title,
+                authors,
+                publish_year,
+                isbn,
+                pub_info,
+                status,
+                "",
+                "",
+                date_read,
+                entry_date,
+                book.pages or "",
+                "",
+            ]
+        )
+    return rows
+
+
+def goodreads_csv_rows(user) -> list[list]:
+    from books.models import Book
+
+    books = Book.objects.filter(scrobble__user=user).distinct().order_by("title")
+
+    rows = []
+    for book in books:
+        user_scrobbles = book.scrobble_set.filter(user=user)
+        last_scrobble = user_scrobbles.order_by("-timestamp").first()
+        first_scrobble = user_scrobbles.order_by("timestamp").first()
+        finished = bool(last_scrobble and last_scrobble.long_play_complete)
+
+        author_list = list(book.authors.all())
+        primary_author = author_list[0].name if author_list else ""
+        author_l_f = format_author_last_first(primary_author) if author_list else ""
+        additional_authors = ", ".join(a.name for a in author_list[1:])
+
+        publish_year = (
+            book.publish_date.year
+            if book.publish_date
+            else (book.first_publish_year or "")
+        )
+        date_read = ""
+        if finished and last_scrobble.timestamp:
+            date_read = last_scrobble.timestamp.strftime("%Y/%m/%d")
+        date_added = ""
+        if first_scrobble and first_scrobble.timestamp:
+            date_added = first_scrobble.timestamp.strftime("%Y/%m/%d")
+        bookshelves = ", ".join(sorted(book.tags.names()))
+        exclusive_shelf = "read" if finished else "currently-reading"
+
+        rows.append(
+            [
+                book.uuid,
+                book.title,
+                primary_author,
+                author_l_f,
+                additional_authors,
+                book.isbn_10 or "",
+                book.isbn_13 or "",
+                "",
+                "",
+                book.publisher or "",
+                "",
+                book.pages or "",
+                publish_year,
+                book.first_publish_year or "",
+                date_read,
+                date_added,
+                bookshelves,
+                "",
+                exclusive_shelf,
+                "",
+                "",
+                "",
+                user_scrobbles.count(),
+                "",
+                "",
+                "0",
+                "",
+                "",
+                "",
+                "",
+                "",
+            ]
+        )
+    return rows
+
 
 def parse_readcomicsonline_uri(uri: str) -> tuple:
     try:

vrobbler/apps/books/views.py

@@ -1,10 +1,17 @@
-from django.http import HttpResponseRedirect
+import csv
-from django.urls import reverse
-from django.views import View
-from django.views import generic
-from books.models import Book, Author, Paper
 
-from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView
+from books.models import Author, Book, Paper
+from books.utils import (
+    GOODREADS_HEADERS,
+    LIBRARYTHING_HEADERS,
+    goodreads_csv_rows,
+    librarything_csv_rows,
+)
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.http import HttpResponse, HttpResponseRedirect
+from django.urls import reverse
+from django.views import View, generic
+from scrobbles.views import ScrobbleableDetailView, ScrobbleableListView
 
 
 class BookListView(ScrobbleableListView):
@@ -39,3 +46,31 @@ class PaperUploadPdfView(View):
             paper.pdf_file.save(pdf_file.name, pdf_file)
 
         return HttpResponseRedirect(reverse("books:paper_detail", args=[slug]))
+
+
+class LibraryThingExportView(LoginRequiredMixin, View):
+    def get(self, request):
+        rows = librarything_csv_rows(request.user)
+
+        response = HttpResponse(content_type="text/csv")
+        response[
+            "Content-Disposition"
+        ] = 'attachment; filename="librarything-books.csv"'
+        writer = csv.writer(response)
+        writer.writerow(LIBRARYTHING_HEADERS)
+        for row in rows:
+            writer.writerow(row)
+        return response
+
+
+class GoodreadsExportView(LoginRequiredMixin, View):
+    def get(self, request):
+        rows = goodreads_csv_rows(request.user)
+
+        response = HttpResponse(content_type="text/csv")
+        response["Content-Disposition"] = 'attachment; filename="goodreads-books.csv"'
+        writer = csv.writer(response)
+        writer.writerow(GOODREADS_HEADERS)
+        for row in rows:
+            writer.writerow(row)
+        return response

vrobbler/templates/books/book_list.html

@@ -12,6 +12,19 @@
 {% endblock  %}
 
 {% block lists %}
+<div class="row">
+    <div class="col-md">
+        <div class="mb-3 btn-group">
+            <button type="button" class="btn btn-sm btn-outline-secondary dropdown-toggle" data-bs-toggle="dropdown" aria-expanded="false">
+                Export
+            </button>
+            <ul class="dropdown-menu">
+                <li><a class="dropdown-item" href="{% url 'books:librarything_export' %}">LibraryThing</a></li>
+                <li><a class="dropdown-item" href="{% url 'books:goodreads_export' %}">Goodreads</a></li>
+            </ul>
+        </div>
+    </div>
+</div>
 <div class="row">
 
     <div class="col-md">