[books] Fix amazon scrapper to use actual AWS endpoints

PROJECT.org

@@ -88,7 +88,7 @@ fetching and simple saving.
 *** Metadata sources
 **** Scraper
 
-* Backlog [0/19] :vrobbler:project:personal:
+* Backlog [1/20] :vrobbler:project:personal:
 ** TODO [#C] Create small utility to clean up tracks scrobbled with wonky playback times :bug:music:scrobbles:
 :PROPERTIES:
 :ID:       702462cf-d54b-48c6-8a7c-78b8de751deb
@@ -578,6 +578,20 @@ named constants for maintainability.
   - ~vrobbler/apps/webpages/models.py~ (line 290) -- ="url"=
   - ~vrobbler/apps/scrobbles/importers/tsv.py~ (line 55) -- ="S"= completion status
 
+    
+** DONE [#A] Fix Amazon book scraper :amazon:scraper:broken:
+:PROPERTIES:
+:ID:       c38aba25-0171-49ab-a9f3-acf2003da429
+:END:
+
+*** Description
+
+File: ~vrobbler/apps/books/amazon.py~ (line 56)
+
+The =scrape_data_from_amazon()= function is likely broken due to Amazon blocking
+scrapers and changing HTML structure. Needs rewrite or replacement with a proper
+API.
+
 * Version 53.1 [1/1]
 ** DONE [#A] Error with loading logdict :scrobbles:bug:logdata:
 :PROPERTIES:

poetry.lock

@@ -4270,6 +4270,29 @@ six = "*"
 [package.extras]
 testing = ["filelock"]
 
+[[package]]
+name = "python-amazon-paapi"
+version = "6.3.0"
+description = "Amazon Product Advertising API 5.0 wrapper for Python"
+optional = false
+python-versions = ">=3.9"
+groups = ["main"]
+files = [
+    {file = "python_amazon_paapi-6.3.0-py3-none-any.whl", hash = "sha256:b7cd852084a49d53c3ba2195531fccbc8c7f4124b2e82e2fda02b53d3b8de521"},
+    {file = "python_amazon_paapi-6.3.0.tar.gz", hash = "sha256:e525d69efcbe4f9566ec2b9b43fa3183c484d166d3852edb38b4df9c0b19cf1f"},
+]
+
+[package.dependencies]
+certifi = ">=2023.0.0"
+pydantic = ">=2.0.0"
+python-dateutil = ">=2.8.0"
+requests = ">=2.28.0"
+six = ">=1.16.0"
+urllib3 = ">=1.26.0,<3"
+
+[package.extras]
+async = ["httpx (>=0.27.0)", "typing-extensions (>=4.15.0)"]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -6032,4 +6055,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.11,<3.15"
-content-hash = "cc5b3b44071d6b0ab4f05189580232cc129b4ed694ab3f0673c3d838c3af0f8a"
+content-hash = "aafab54d3c3d674b917782bf449b7d6324ca2259fb58bff13a08caabe110c342"

pyproject.toml

@@ -64,6 +64,7 @@ fitparse = "^1.2.0"
 lxml = ">=5.5.0"
 vaderSentiment = "^3.3.2"
 sqids = "^0.5.2"
+python-amazon-paapi = "^6.3.0"
 
 [tool.poetry.group.test]
 optional = true

vrobbler/apps/books/amazon.py

@@ -1,128 +0,0 @@
-from enum import Enum
-from typing import Optional
-from bs4 import BeautifulSoup
-import requests
-import logging
-
-logger = logging.getLogger(__name__)
-
-USER_AGENT = "Mozilla/5.0 (Android 4.4; Mobile; rv:41.0) Gecko/41.0 Firefox/41.0"
-AMAZON_SEARCH_URL = "https://www.amazon.com/s?k={amazon_id}"
-
-
-class AmazonAttribute(Enum):
-    SERIES = 0
-    PAGES = 1
-    LANGUAGE = 2
-    PUBLISHER = 3
-    PUB_DATE = 4
-    DIMENSIONS = 5
-    ISBN_10 = 6
-    ISBN_13 = 7
-
-
-def strip_and_clean(text):
-    return text.strip("\n").rstrip().lstrip()
-
-
-def get_rating_from_soup(soup) -> Optional[int]:
-    rating = None
-    try:
-        potential_rating = soup.find("div", class_="allmusic-rating")
-        if potential_rating:
-            rating = int(strip_and_clean(potential_rating.get_text()))
-    except ValueError:
-        pass
-    return rating
-
-
-def get_review_from_soup(soup) -> str:
-    review = ""
-    try:
-        potential_text = soup.find("div", class_="text")
-        if potential_text:
-            review = strip_and_clean(potential_text.get_text())
-    except ValueError:
-        pass
-    return review
-
-
-def scrape_data_from_amazon(url) -> dict:
-    data_dict = {}
-    headers = {"User-Agent": USER_AGENT}
-    r = requests.get(url, headers=headers)
-    if r.status_code == 200:
-        soup = BeautifulSoup(r.text, "html.parser")
-        # TODO Fix this scraper
-        data_dict["rating"] = get_rating_from_soup(soup)
-        data_dict["review"] = get_review_from_soup(soup)
-    return data_dict
-
-
-def get_amazon_product_dict(amazon_id: str) -> dict:
-    data_dict = {}
-    url = ""
-
-    search_url = AMAZON_SEARCH_URL.format(amazon_id=amazon_id)
-    headers = {
-        "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
-        "accept-language": "en-GB,en;q=0.9",
-    }
-
-    response = requests.get(search_url, headers=headers)
-
-    if response.status_code != 200:
-        logger.info(f"Bad http response from Amazon {response}")
-        return data_dict
-
-    soup = BeautifulSoup(response.text, "html.parser")
-    results = soup.find("a", class_="a-link-normal")
-
-    if not results:
-        logger.info(f"No search results for {amazon_id}")
-        return data_dict
-
-    product_url = "https://www.amazon.com" + str(results.get("href", ""))
-
-    data_dict = {}
-    response = requests.get(product_url, headers=headers)
-
-    if response.status_code != 200:
-        logger.info(f"Bad http response from Amazon {response}")
-        return data_dict
-
-    soup = BeautifulSoup(response.text, "html.parser")
-    try:
-        data_dict["title"] = soup.findAll("span", class_="celwidget")[1].text.strip()
-        data_dict["cover_url"] = soup.find("img", class_="frontImage").get("src")
-        data_dict["summary"] = soup.findAll("div", class_="a-expander-content")[1].text
-        meta = soup.findAll("div", class_="rpi-attribute-value")
-        data_dict["isbn"] = meta[AmazonAttribute.ISBN_10.value].text.strip()
-        pages = meta[AmazonAttribute.PAGES.value].text
-        if "pages" in pages:
-            data_dict["pages"] = (
-                meta[AmazonAttribute.PAGES.value].text.split("pages")[0].strip()
-            )
-    except IndexError as e:
-        logger.error(f"Amazon lookup is failing for this product {amazon_id}: {e}")
-    except AttributeError as e:
-        logger.error(f"Amazon lookup is failing for this product {amazon_id}: {e}")
-
-    return data_dict
-
-
-def lookup_book_from_amazon(amazon_id: str) -> dict:
-    top = {}
-
-    return {
-        "title": top.get("title"),
-        "isbn": isbn,
-        "openlibrary_id": ol_id,
-        "goodreads_id": get_first("id_goodreads", top),
-        "first_publish_year": top.get("first_publish_year"),
-        "first_sentence": first_sentence,
-        "pages": top.get("number_of_pages_median", None),
-        "cover_url": COVER_URL.format(id=ol_id),
-        "ol_author_id": ol_author_id,
-        "subject_key_list": top.get("subject_key", []),
-    }

vrobbler/apps/books/constants.py

@@ -17,6 +17,7 @@ class MediaSourceTag(str, Enum):
     LOCG = "source_locg"
     KOREADER = "source_koreader"
     SEMANTIC_SCHOLAR = "source_semantic_scholar"
+    AMAZON = "source_amazon"
 
     @classmethod
     def choices(cls):

vrobbler/apps/books/models.py

@@ -23,6 +23,7 @@ from books.sources.comicvine import (
     lookup_issue_by_comicvine_id,
 )
 from books.sources.google import lookup_book_from_google
+from books.sources.amazon import lookup_book_from_amazon
 from books.sources.openlibrary import (
     lookup_book_from_openlibrary as lookup_book_from_ol,
 )
@@ -260,6 +261,7 @@ class Book(LongPlayScrobblableMixin):
         url: str = "",
         enrich: bool = True,
         commit: bool = True,
+        amazon_id: str | None = None,
     ):
         """Given a title, get a Book instance.
 
@@ -321,6 +323,13 @@ class Book(LongPlayScrobblableMixin):
                         book_dict.setdefault(k, v)
                 source_tag = MediaSourceTag.COMICVINE
 
+        # Try Amazon PAAPI as a fallback when given an ASIN
+        if amazon_id and not book_dict:
+            amazon_data = lookup_book_from_amazon(amazon_id)
+            if amazon_data:
+                book_dict.update(amazon_data)
+                source_tag = MediaSourceTag.AMAZON
+
         if not book_dict:
             logger.warning(
                 "No book found in any source, using data as is",

vrobbler/apps/books/sources/amazon.py

@@ -0,0 +1,123 @@
+import logging
+import warnings
+
+from django.conf import settings
+
+with warnings.catch_warnings():
+    warnings.simplefilter("ignore", DeprecationWarning)
+    from amazon_paapi import AmazonApi
+
+logger = logging.getLogger(__name__)
+
+_amazon_client = None
+
+
+def _get_client() -> AmazonApi | None:
+    global _amazon_client
+    if _amazon_client is not None:
+        return _amazon_client
+
+    key = settings.AMAZON_PAAPI_ACCESS_KEY
+    secret = settings.AMAZON_PAAPI_SECRET_KEY
+    tag = settings.AMAZON_PAAPI_ASSOCIATE_TAG
+    country = settings.AMAZON_PAAPI_COUNTRY
+
+    if not all([key, secret, tag]):
+        logger.warning("Amazon PAAPI credentials not configured")
+        return None
+
+    _amazon_client = AmazonApi(key, secret, tag, country)
+    return _amazon_client
+
+
+def lookup_book_from_amazon(asin: str) -> dict:
+    book_dict: dict = {}
+
+    client = _get_client()
+    if not client:
+        return book_dict
+
+    try:
+        items = client.get_items(
+            items=[asin],
+            Condition="New",
+            LanguagesOfPreference=["en_US"],
+        )
+    except Exception as e:
+        logger.warning(f"Amazon PAAPI lookup failed for {asin}: {e}")
+        return book_dict
+
+    if not items:
+        logger.info(f"No Amazon item found for {asin}")
+        return book_dict
+
+    item = items[0]
+    raw = item.to_dict()
+    item_info = raw.get("item_info", {}) or {}
+
+    book_dict["title"] = _get_nested(item_info, "title", "display_value")
+    if not book_dict.get("title"):
+        book_dict["title"] = _get_nested(item_info, "title", "value")
+
+    contributors = _get_nested(item_info, "by_line_info", "contributors") or []
+    authors = [
+        c["name"]
+        for c in contributors
+        if c.get("role", "").lower() in ("author", "artist", "writer")
+    ]
+    if authors:
+        book_dict["authors"] = authors
+
+    publisher = _get_nested(item_info, "by_line_info", "manufacturer")
+    if publisher:
+        book_dict["publisher"] = publisher
+
+    isb_ns = _get_nested(item_info, "external_ids", "isb_ns")
+    if isb_ns and isinstance(isb_ns, list):
+        for isb in isb_ns:
+            if isinstance(isb, dict):
+                if isb.get("type") == "ISBN_13":
+                    book_dict["isbn_13"] = isb.get("value")
+                elif isb.get("type") == "ISBN_10":
+                    book_dict["isbn_10"] = isb.get("value")
+
+    pages_count = _get_nested(item_info, "content_info", "pages_count")
+    if pages_count and isinstance(pages_count, dict):
+        book_dict["pages"] = pages_count.get("value") or pages_count.get("display_value")
+
+    languages = _get_nested(item_info, "content_info", "languages") or []
+    if languages and isinstance(languages, list):
+        lang = languages[0]
+        if isinstance(lang, dict):
+            book_dict["language"] = lang.get("display_value") or lang.get("value")
+
+    pub_date = _get_nested(item_info, "content_info", "publication_date")
+    if not pub_date:
+        pub_date = _get_nested(item_info, "product_info", "release_date")
+    if pub_date and isinstance(pub_date, dict):
+        book_dict["publish_date"] = pub_date.get("display_value") or pub_date.get("value")
+
+    features = item_info.get("features") or []
+    if features and isinstance(features, list):
+        book_dict["summary"] = " ".join(features[:5])
+
+    images = raw.get("images", {}) or {}
+    primary = images.get("primary", {}) or {}
+    for size in ("large", "hi_res", "medium"):
+        candidate = primary.get(size, {}) or {}
+        url = candidate.get("url")
+        if url:
+            book_dict["cover_url"] = url
+            break
+
+    book_dict["detail_page_url"] = raw.get("detail_page_url")
+
+    return book_dict
+
+
+def _get_nested(d: dict, *keys):
+    for key in keys:
+        if not isinstance(d, dict):
+            return None
+        d = d.get(key)
+    return d

vrobbler/settings.py

@@ -82,6 +82,11 @@ TODOIST_CLIENT_SECRET = os.getenv("VROBBLER_TODOIST_CLIENT_SECRET", "")
 GOOGLE_API_KEY = os.getenv("VROBBLER_GOOGLE_API_KEY", "")
 LICHESS_API_KEY = os.getenv("VROBBLER_LICHESS_API_KEY", "")
 
+AMAZON_PAAPI_ACCESS_KEY = os.getenv("VROBBLER_AMAZON_PAAPI_ACCESS_KEY", "")
+AMAZON_PAAPI_SECRET_KEY = os.getenv("VROBBLER_AMAZON_PAAPI_SECRET_KEY", "")
+AMAZON_PAAPI_ASSOCIATE_TAG = os.getenv("VROBBLER_AMAZON_PAAPI_ASSOCIATE_TAG", "")
+AMAZON_PAAPI_COUNTRY = os.getenv("VROBBLER_AMAZON_PAAPI_COUNTRY", "US")
+
 DEFAULT_TASK_CONTEXT_TAGS = [
     "Dev",
     "Home",