diff --git a/PROJECT.org b/PROJECT.org index 89b8623..0a7a868 100644 --- a/PROJECT.org +++ b/PROJECT.org @@ -524,23 +524,6 @@ AllTrails is the best source, though having TrailForks is nice to. Would be nice to have some loose connection to the actual event in my Garmin profile. -** TODO [#B] Fix how we show notes and descriptions from scrobbles to users :metadata:notes:tasks: -:PROPERTIES: -:ID: adf4c513-a417-4ec9-8831-f01ffcf63276 -:END: -*** Description - -Currently the display of notes leaves something to be desired. The biggest issue -is that they don't look good on mobile and are probably trying to be too cute. -Rather than post-it note style, we should just put notes in a list under the -description, above the Edit Log toggle, with timestamps for when they were -added. - -They should also probably support markdown formatting and that should be -displayed in the template. - -** TODO [#B] Add CSV endpoint for book scrobbles that LibraryThing can ingest :books:feature:export: -https://app.todoist.com/app/task/add-a-csv-endpoint-for-users-book-reads-that-library-thing-can-ingest-6X7QPMRp265xMXqg#comment-6X7QrXq6gJjMP4hg ** TODO [#B] Make IMAP and WebDAV configurable :webdav:feature:imap:importers: :PROPERTIES: :ID: b1426d92-2feb-4d15-9738-d5b7b0594f96 @@ -611,6 +594,32 @@ The Edit log form should have from top to bottom: - People (which should be similar to the Bird widget on BirdLocation and allow setting per user score, win true/false, rank, new true/false, seat_ordrer) - Expansion ids (which should a multi-select widget of expansions for this game) - Location (which should be a drop down of BoardGameLocations for this user) +** DONE [#B] Add CSV endpoint for book scrobbles that LibraryThing can ingest :books:feature:export: +:PROPERTIES: +:ID: fd8a2778-ff46-394b-9868-edb9a48d0ce6 +:END: + + +*** Description +We should provide an "Export for LibraryThings" link on the Books list page +which creates a CSV file of all books and tags for "in-progress" or "finished" +depending on whether the book has been marked as finished. + +[[https://www.librarything.com/LibraryThingSample.csv][Sample CSV file]] + +*** Implementation + +Added a `books:librarything_export` endpoint (a `LoginRequiredMixin` view) that +streams a CSV matching the [[https://www.librarything.com/LibraryThingSample.csv][LibraryThing sample]] +column layout (TITLE, AUTHOR (last, first), DATE, ISBN, PUBLICATION INFO, TAGS, +RATING, REVIEW, DATE READ, PAGE COUNT, CALL NUMBER). The `TAGS` column is set to +"in-progress" or "finished" based on the user's last scrobble's +`long_play_complete` flag, `DATE READ` reflects that scrobble's timestamp when +finished, author names are reformatted to "last, first", and the export is +limited to books the user has scrobbled. The CSV logic lives in +`books.utils.librarything_csv_rows()` with a `format_author_last_first()` +helper; a link on the Books list page (`book_list.html`) triggers the download. + ** DONE [#A] Fix recursive bug in backup script :backups:bug: :PROPERTIES: :ID: e986d35e-fc23-5d31-9d2a-0a95da4748a2 diff --git a/tests/books_tests/__init__.py b/tests/books_tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/books_tests/test_goodreads_export.py b/tests/books_tests/test_goodreads_export.py new file mode 100644 index 0000000..71a77b5 --- /dev/null +++ b/tests/books_tests/test_goodreads_export.py @@ -0,0 +1,133 @@ +import csv +from datetime import datetime +from io import StringIO + +import pytest +from books.models import Author, Book +from books.utils import GOODREADS_HEADERS +from django.contrib.auth import get_user_model +from django.urls import reverse +from django.utils import timezone +from scrobbles.models import Scrobble + +User = get_user_model() + + +@pytest.fixture +def user(): + return User.objects.create_user( + username="reader", email="reader@example.com", password="testpass" + ) + + +@pytest.fixture +def book(user): + primary = Author.objects.create(name="Nicholas A. Basbanes") + secondary = Author.objects.create(name="Jane Q. Doe") + book = Book.objects.create( + title="A Gentle Madness", + isbn_13="9780805061765", + isbn_10="0805061762", + pages=500, + publisher="Holt Paperbacks", + first_publish_year=1999, + ) + book.authors.add(primary, secondary) + book.tags.add("history", "books") + return book + + +def test_goodreads_export_requires_login(client): + url = reverse("books:goodreads_export") + response = client.get(url) + assert response.status_code == 302 + + +@pytest.mark.django_db +def test_goodreads_export_empty(client, user): + client.force_login(user) + url = reverse("books:goodreads_export") + response = client.get(url) + assert response.status_code == 200 + assert response["Content-Type"] == "text/csv" + rows = list(csv.reader(StringIO(response.content.decode()))) + assert rows[0] == GOODREADS_HEADERS + assert len(rows) == 1 + + +@pytest.mark.django_db +def test_goodreads_export_marks_in_progress(client, user, book): + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 3, 1)), + long_play_complete=False, + ) + + client.force_login(user) + url = reverse("books:goodreads_export") + response = client.get(url) + rows = list(csv.DictReader(StringIO(response.content.decode()))) + assert len(rows) == 1 + assert rows[0]["Title"] == "A Gentle Madness" + assert rows[0]["Author"] == "Nicholas A. Basbanes" + assert rows[0]["Author l-f"] == "Basbanes, Nicholas A." + assert rows[0]["Additional Authors"] == "Jane Q. Doe" + assert rows[0]["ISBN"] == "0805061762" + assert rows[0]["ISBN13"] == "9780805061765" + assert rows[0]["Number of Pages"] == "500" + assert rows[0]["Year Published"] == "1999" + assert rows[0]["Exclusive Shelf"] == "currently-reading" + assert rows[0]["Date Read"] == "" + assert rows[0]["Date Added"] == "2023/03/01" + assert rows[0]["Bookshelves"] == "books, history" + assert rows[0]["Read Count"] == "1" + assert rows[0]["Owned Copies"] == "0" + + +@pytest.mark.django_db +def test_goodreads_export_marks_finished(client, user, book): + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 5, 5)), + long_play_complete=True, + ) + + client.force_login(user) + url = reverse("books:goodreads_export") + response = client.get(url) + rows = list(csv.DictReader(StringIO(response.content.decode()))) + assert len(rows) == 1 + assert rows[0]["Exclusive Shelf"] == "read" + assert rows[0]["Date Read"] == "2023/05/05" + + +@pytest.mark.django_db +def test_goodreads_export_date_added_is_first_scrobble(client, user, book): + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 3, 1)), + long_play_complete=False, + ) + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 5, 5)), + long_play_complete=True, + ) + + client.force_login(user) + url = reverse("books:goodreads_export") + response = client.get(url) + rows = list(csv.DictReader(StringIO(response.content.decode()))) + assert len(rows) == 1 + assert rows[0]["Exclusive Shelf"] == "read" + assert rows[0]["Date Added"] == "2023/03/01" + assert rows[0]["Date Read"] == "2023/05/05" + assert rows[0]["Read Count"] == "2" diff --git a/tests/books_tests/test_librarything_export.py b/tests/books_tests/test_librarything_export.py new file mode 100644 index 0000000..bbcb67e --- /dev/null +++ b/tests/books_tests/test_librarything_export.py @@ -0,0 +1,135 @@ +import csv +from datetime import datetime +from io import StringIO + +import pytest +from books.models import Author, Book +from django.contrib.auth import get_user_model +from django.urls import reverse +from django.utils import timezone +from scrobbles.models import Scrobble + +User = get_user_model() + + +@pytest.fixture +def user(): + return User.objects.create_user( + username="reader", email="reader@example.com", password="testpass" + ) + + +@pytest.fixture +def book(user): + author = Author.objects.create(name="Nicholas A. Basbanes") + book = Book.objects.create( + title="A Gentle Madness", + isbn_13="0805061762", + pages=500, + publisher="Holt Paperbacks", + first_publish_year=1999, + ) + book.authors.add(author) + return book + + +def test_librarything_export_requires_login(client): + url = reverse("books:librarything_export") + response = client.get(url) + assert response.status_code == 302 + + +@pytest.mark.django_db +def test_librarything_export_empty(client, user): + client.force_login(user) + url = reverse("books:librarything_export") + response = client.get(url) + assert response.status_code == 200 + assert response["Content-Type"] == "text/csv" + rows = list(csv.reader(StringIO(response.content.decode()))) + assert rows[0] == [ + "TITLE", + "AUTHOR (last, first)", + "DATE", + "ISBN", + "PUBLICATION INFO", + "TAGS", + "RATING", + "REVIEW", + "DATE READ", + "ENTRY DATE", + "PAGE COUNT", + "CALL NUMBER", + ] + assert len(rows) == 1 + + +@pytest.mark.django_db +def test_librarything_export_marks_in_progress(client, user, book): + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 3, 1)), + long_play_complete=False, + ) + + client.force_login(user) + url = reverse("books:librarything_export") + response = client.get(url) + rows = list(csv.DictReader(StringIO(response.content.decode()))) + assert len(rows) == 1 + assert rows[0]["TITLE"] == "A Gentle Madness" + assert rows[0]["AUTHOR (last, first)"] == "Basbanes, Nicholas A." + assert rows[0]["ISBN"] == "0805061762" + assert rows[0]["PAGE COUNT"] == "500" + assert rows[0]["TAGS"] == "in-progress" + assert rows[0]["DATE READ"] == "" + assert rows[0]["ENTRY DATE"] == "2023-03-01" + + +@pytest.mark.django_db +def test_librarything_export_marks_finished(client, user, book): + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 5, 5)), + long_play_complete=True, + ) + + client.force_login(user) + url = reverse("books:librarything_export") + response = client.get(url) + rows = list(csv.DictReader(StringIO(response.content.decode()))) + assert len(rows) == 1 + assert rows[0]["TAGS"] == "finished" + assert rows[0]["DATE READ"] == "2023-05-05" + assert rows[0]["ENTRY DATE"] == "2023-05-05" + + +@pytest.mark.django_db +def test_librarything_export_entry_date_is_first_scrobble(client, user, book): + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 3, 1)), + long_play_complete=False, + ) + Scrobble.objects.create( + book=book, + media_type=Scrobble.MediaType.BOOK, + user=user, + timestamp=timezone.make_aware(datetime(2023, 5, 5)), + long_play_complete=True, + ) + + client.force_login(user) + url = reverse("books:librarything_export") + response = client.get(url) + rows = list(csv.DictReader(StringIO(response.content.decode()))) + assert len(rows) == 1 + assert rows[0]["TAGS"] == "finished" + assert rows[0]["ENTRY DATE"] == "2023-03-01" + assert rows[0]["DATE READ"] == "2023-05-05" diff --git a/vrobbler/apps/books/urls.py b/vrobbler/apps/books/urls.py index fdb051a..c5ec033 100644 --- a/vrobbler/apps/books/urls.py +++ b/vrobbler/apps/books/urls.py @@ -1,5 +1,5 @@ -from django.urls import path from books import views +from django.urls import path app_name = "books" @@ -27,4 +27,14 @@ urlpatterns = [ views.PaperUploadPdfView.as_view(), name="paper_upload_pdf", ), + path( + "books/export/librarything/", + views.LibraryThingExportView.as_view(), + name="librarything_export", + ), + path( + "books/export/goodreads/", + views.GoodreadsExportView.as_view(), + name="goodreads_export", + ), ] diff --git a/vrobbler/apps/books/utils.py b/vrobbler/apps/books/utils.py index 5932dda..6f905cb 100644 --- a/vrobbler/apps/books/utils.py +++ b/vrobbler/apps/books/utils.py @@ -3,6 +3,186 @@ from urllib.parse import urlparse, urlunparse from titlecase import titlecase +LIBRARYTHING_HEADERS = [ + "TITLE", + "AUTHOR (last, first)", + "DATE", + "ISBN", + "PUBLICATION INFO", + "TAGS", + "RATING", + "REVIEW", + "DATE READ", + "ENTRY DATE", + "PAGE COUNT", + "CALL NUMBER", +] + +GOODREADS_HEADERS = [ + "Book Id", + "Title", + "Author", + "Author l-f", + "Additional Authors", + "ISBN", + "ISBN13", + "My Rating", + "Average Rating", + "Publisher", + "Binding", + "Number of Pages", + "Year Published", + "Original Publication Year", + "Date Read", + "Date Added", + "Bookshelves", + "Bookshelves with positions", + "Exclusive Shelf", + "My Review", + "Spoiler", + "Private Notes", + "Read Count", + "Recommended For", + "Recommended By", + "Owned Copies", + "Original Purchase Date", + "Original Purchase Location", + "Condition", + "Condition Description", + "BCID", +] + + +def format_author_last_first(name: str) -> str: + parts = name.strip().split() + if len(parts) < 2: + return name + return f"{parts[-1]}, {' '.join(parts[:-1])}" + + +def librarything_csv_rows(user) -> list[list]: + from books.models import Book + + books = Book.objects.filter(scrobble__user=user).distinct().order_by("title") + + rows = [] + for book in books: + user_scrobbles = book.scrobble_set.filter(user=user) + last_scrobble = user_scrobbles.order_by("-timestamp").first() + first_scrobble = user_scrobbles.order_by("timestamp").first() + finished = bool(last_scrobble and last_scrobble.long_play_complete) + status = "finished" if finished else "in-progress" + + authors = ", ".join( + format_author_last_first(a.name) for a in book.authors.all() + ) + publish_year = ( + book.publish_date.year + if book.publish_date + else (book.first_publish_year or "") + ) + isbn = book.isbn_13 or book.isbn_10 or "" + pub_info = "" + if book.publisher: + pub_info = book.publisher + if publish_year: + pub_info = f"{pub_info} ({publish_year})".strip() + if book.pages: + pub_info = f"{pub_info}, {book.pages} pages" + date_read = "" + if finished and last_scrobble.timestamp: + date_read = last_scrobble.timestamp.strftime("%Y-%m-%d") + entry_date = "" + if first_scrobble and first_scrobble.timestamp: + entry_date = first_scrobble.timestamp.strftime("%Y-%m-%d") + + rows.append( + [ + book.title, + authors, + publish_year, + isbn, + pub_info, + status, + "", + "", + date_read, + entry_date, + book.pages or "", + "", + ] + ) + return rows + + +def goodreads_csv_rows(user) -> list[list]: + from books.models import Book + + books = Book.objects.filter(scrobble__user=user).distinct().order_by("title") + + rows = [] + for book in books: + user_scrobbles = book.scrobble_set.filter(user=user) + last_scrobble = user_scrobbles.order_by("-timestamp").first() + first_scrobble = user_scrobbles.order_by("timestamp").first() + finished = bool(last_scrobble and last_scrobble.long_play_complete) + + author_list = list(book.authors.all()) + primary_author = author_list[0].name if author_list else "" + author_l_f = format_author_last_first(primary_author) if author_list else "" + additional_authors = ", ".join(a.name for a in author_list[1:]) + + publish_year = ( + book.publish_date.year + if book.publish_date + else (book.first_publish_year or "") + ) + date_read = "" + if finished and last_scrobble.timestamp: + date_read = last_scrobble.timestamp.strftime("%Y/%m/%d") + date_added = "" + if first_scrobble and first_scrobble.timestamp: + date_added = first_scrobble.timestamp.strftime("%Y/%m/%d") + bookshelves = ", ".join(sorted(book.tags.names())) + exclusive_shelf = "read" if finished else "currently-reading" + + rows.append( + [ + book.uuid, + book.title, + primary_author, + author_l_f, + additional_authors, + book.isbn_10 or "", + book.isbn_13 or "", + "", + "", + book.publisher or "", + "", + book.pages or "", + publish_year, + book.first_publish_year or "", + date_read, + date_added, + bookshelves, + "", + exclusive_shelf, + "", + "", + "", + user_scrobbles.count(), + "", + "", + "0", + "", + "", + "", + "", + "", + ] + ) + return rows + def parse_readcomicsonline_uri(uri: str) -> tuple: try: diff --git a/vrobbler/apps/books/views.py b/vrobbler/apps/books/views.py index 0eaef44..e50c5d1 100644 --- a/vrobbler/apps/books/views.py +++ b/vrobbler/apps/books/views.py @@ -1,10 +1,17 @@ -from django.http import HttpResponseRedirect -from django.urls import reverse -from django.views import View -from django.views import generic -from books.models import Book, Author, Paper +import csv -from scrobbles.views import ScrobbleableListView, ScrobbleableDetailView +from books.models import Author, Book, Paper +from books.utils import ( + GOODREADS_HEADERS, + LIBRARYTHING_HEADERS, + goodreads_csv_rows, + librarything_csv_rows, +) +from django.contrib.auth.mixins import LoginRequiredMixin +from django.http import HttpResponse, HttpResponseRedirect +from django.urls import reverse +from django.views import View, generic +from scrobbles.views import ScrobbleableDetailView, ScrobbleableListView class BookListView(ScrobbleableListView): @@ -39,3 +46,31 @@ class PaperUploadPdfView(View): paper.pdf_file.save(pdf_file.name, pdf_file) return HttpResponseRedirect(reverse("books:paper_detail", args=[slug])) + + +class LibraryThingExportView(LoginRequiredMixin, View): + def get(self, request): + rows = librarything_csv_rows(request.user) + + response = HttpResponse(content_type="text/csv") + response[ + "Content-Disposition" + ] = 'attachment; filename="librarything-books.csv"' + writer = csv.writer(response) + writer.writerow(LIBRARYTHING_HEADERS) + for row in rows: + writer.writerow(row) + return response + + +class GoodreadsExportView(LoginRequiredMixin, View): + def get(self, request): + rows = goodreads_csv_rows(request.user) + + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = 'attachment; filename="goodreads-books.csv"' + writer = csv.writer(response) + writer.writerow(GOODREADS_HEADERS) + for row in rows: + writer.writerow(row) + return response diff --git a/vrobbler/templates/books/book_list.html b/vrobbler/templates/books/book_list.html index b62d977..5e5255d 100644 --- a/vrobbler/templates/books/book_list.html +++ b/vrobbler/templates/books/book_list.html @@ -12,6 +12,19 @@ {% endblock %} {% block lists %} +
+
+
+ + +
+
+