Scrape all the things

vrobbler/apps/music/allmusic.py

@@ -0,0 +1,85 @@
+import urllib
+from typing import Optional
+from bs4 import BeautifulSoup
+import requests
+import logging
+
+logger = logging.getLogger(__name__)
+
+ALLMUSIC_SEARCH_URL = "https://www.allmusic.com/search/{subpath}/{query}"
+
+
+def strip_and_clean(text):
+    return text.strip("\n").rstrip().lstrip()
+
+
+def get_rating_from_soup(soup) -> Optional[int]:
+    rating = None
+    try:
+        potential_rating = soup.find("div", class_="allmusic-rating")
+        if potential_rating:
+            rating = int(strip_and_clean(potential_rating.get_text()))
+    except ValueError:
+        pass
+    return rating
+
+
+def get_review_from_soup(soup) -> str:
+    review = ""
+    try:
+        potential_text = soup.find("div", class_="text")
+        if potential_text:
+            review = strip_and_clean(potential_text.get_text())
+    except ValueError:
+        pass
+    return review
+
+
+def scrape_data_from_allmusic(url) -> dict:
+    data_dict = {}
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+    if r.status_code == 200:
+        soup = BeautifulSoup(r.text, "html")
+        data_dict["rating"] = get_rating_from_soup(soup)
+        data_dict["review"] = get_review_from_soup(soup)
+    return data_dict
+
+
+def get_allmusic_slug(artist_name=None, album_name=None) -> str:
+    slug = ""
+    if not artist_name:
+        return slug
+
+    subpath = "artists"
+    class_ = "name"
+    query = urllib.parse.quote(artist_name)
+    if album_name:
+        subpath = "albums"
+        class_ = "title"
+        query = "+".join([query, urllib.parse.quote(album_name)])
+
+    url = ALLMUSIC_SEARCH_URL.format(subpath=subpath, query=query)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+
+    if r.status_code != 200:
+        logger.info(f"Bad http response from Allmusic {r}")
+        return slug
+
+    soup = BeautifulSoup(r.text, "html")
+    results = soup.find("ul", class_="search-results")
+
+    if not results:
+        logger.info(f"No search results for {query}")
+        return slug
+
+    prime_result = results.findAll("div", class_=class_)
+
+    if not prime_result:
+        logger.info(f"Could not find specific result for search {query}")
+
+    result_url = prime_result[0].find_all("a")[0]["href"]
+    slug = result_url.split("/")[-1:][0]
+
+    return slug

vrobbler/apps/music/bandcamp.py

@@ -0,0 +1,49 @@
+import logging
+import urllib
+
+import requests
+from bs4 import BeautifulSoup
+
+logger = logging.getLogger(__name__)
+BANDCAMP_SEARCH_URL = "https://bandcamp.com/search?q={query}&item_type={itype}"
+
+
+def get_bandcamp_slug(artist_name=None, album_name=None) -> str:
+    slug = ""
+    if not artist_name:
+        return slug
+
+    query = urllib.parse.quote(artist_name)
+    item_type = "b"
+    class_ = "heading"
+    if album_name:
+        item_type = "a"
+        query = "+".join([query, urllib.parse.quote(album_name)])
+
+    url = BANDCAMP_SEARCH_URL.format(query=query, itype=item_type)
+    headers = {"User-Agent": "Vrobbler 0.11.12"}
+    r = requests.get(url, headers=headers)
+
+    if r.status_code != 200:
+        logger.info(f"Bad http response from Bandcamp {r}")
+        return slug
+
+    soup = BeautifulSoup(r.text, "html")
+
+    results = soup.find("ul", class_="result-items")
+
+    if not results:
+        logger.info(f"No search results for {query}")
+        return slug
+
+    prime_result = results.findAll("div", class_=class_)
+
+    if not prime_result:
+        logger.info(f"Could not find specific result for search {query}")
+
+    result_url = prime_result[0].find_all("a")[0]["href"]
+    if item_type == "b":
+        slug = result_url.split("/")[2].split(".")[0]
+    else:
+        slug = result_url.split("?")[0]
+    return slug

vrobbler/apps/music/migrations/0018_album_allmusic_rating_album_allmusic_review.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-15 01:17
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0017_track_genre"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="allmusic_rating",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+        migrations.AddField(
+            model_name="album",
+            name="allmusic_review",
+            field=models.TextField(blank=True, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0019_artist_allmusic_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.1.5 on 2023-03-15 03:05
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0018_album_allmusic_rating_album_allmusic_review"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="artist",
+            name="allmusic_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

vrobbler/apps/music/migrations/0020_album_bandcamp_id_artist_bandcamp_id.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.1.5 on 2023-03-15 03:32
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("music", "0019_artist_allmusic_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="album",
+            name="bandcamp_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+        migrations.AddField(
+            model_name="artist",
+            name="bandcamp_id",
+            field=models.CharField(blank=True, max_length=100, null=True),
+        ),
+    ]

vrobbler/apps/music/models.py

@@ -14,6 +14,11 @@ from django.utils.translation import gettext_lazy as _
 from django_extensions.db.models import TimeStampedModel
 from scrobbles.mixins import ScrobblableMixin
 from music.theaudiodb import lookup_artist_from_tadb, lookup_album_from_tadb
+from vrobbler.apps.music.allmusic import (
+    get_allmusic_slug,
+    scrape_data_from_allmusic,
+)
+from vrobbler.apps.music.bandcamp import get_bandcamp_slug
 
 logger = logging.getLogger(__name__)
 BNULL = {"blank": True, "null": True}
@@ -26,6 +31,8 @@ class Artist(TimeStampedModel):
     theaudiodb_genre = models.CharField(max_length=255, **BNULL)
     theaudiodb_mood = models.CharField(max_length=255, **BNULL)
     musicbrainz_id = models.CharField(max_length=255, **BNULL)
+    allmusic_id = models.CharField(max_length=100, **BNULL)
+    bandcamp_id = models.CharField(max_length=100, **BNULL)
     thumbnail = models.ImageField(upload_to="artist/", **BNULL)
 
     class Meta:
@@ -38,6 +45,18 @@ class Artist(TimeStampedModel):
     def mb_link(self):
         return f"https://musicbrainz.org/artist/{self.musicbrainz_id}"
 
+    @property
+    def allmusic_link(self):
+        if self.allmusic_id:
+            return f"https://www.allmusic.com/artist/{self.allmusic_id}"
+        return ""
+
+    @property
+    def bandcamp_link(self):
+        if self.bandcamp_id:
+            return f"https://{self.bandcamp_id}.bandcamp.com/"
+        return ""
+
     def get_absolute_url(self):
         return reverse("music:artist_detail", kwargs={"slug": self.uuid})
 
@@ -61,6 +80,24 @@ class Artist(TimeStampedModel):
 
         return ChartRecord.objects.filter(track__artist=self).order_by("-year")
 
+    def scrape_allmusic(self, force=False) -> None:
+        if not self.allmusic_id or force:
+            slug = get_allmusic_slug(self.name)
+            if not slug:
+                logger.info(f"No allmsuic link for {self}")
+                return
+            self.allmusic_id = slug
+            self.save(update_fields=["allmusic_id"])
+
+    def scrape_bandcamp(self, force=False) -> None:
+        if not self.bandcamp_id or force:
+            slug = get_bandcamp_slug(self.name)
+            if not slug:
+                logger.info(f"No bandcamp link for {self}")
+                return
+            self.bandcamp_id = slug
+            self.save(update_fields=["bandcamp_id"])
+
     def fix_metadata(self):
         tadb_info = lookup_artist_from_tadb(self.name)
         if not tadb_info:
@@ -109,6 +146,9 @@ class Album(TimeStampedModel):
     theaudiodb_speed = models.CharField(max_length=255, **BNULL)
     theaudiodb_theme = models.CharField(max_length=255, **BNULL)
     allmusic_id = models.CharField(max_length=255, **BNULL)
+    allmusic_rating = models.IntegerField(**BNULL)
+    allmusic_review = models.TextField(**BNULL)
+    bandcamp_id = models.CharField(max_length=100, **BNULL)
     rateyourmusic_id = models.CharField(max_length=255, **BNULL)
     wikipedia_slug = models.CharField(max_length=255, **BNULL)
     discogs_id = models.CharField(max_length=255, **BNULL)
@@ -139,6 +179,29 @@ class Album(TimeStampedModel):
     def primary_artist(self):
         return self.artists.first()
 
+    def scrape_allmusic(self, force=False) -> None:
+        if not self.allmusic_id or force:
+            slug = get_allmusic_slug(self.name, self.primary_artist.name)
+            if not slug:
+                logger.info(
+                    f"No allmsuic link for {self} by {self.primary_artist}"
+                )
+                return
+            self.allmusic_id = slug
+            self.save(update_fields=["allmusic_id"])
+
+        allmusic_data = scrape_data_from_allmusic(self.allmusic_link)
+
+        if not allmusic_data:
+            logger.info(
+                f"No allmsuic data for {self} by {self.primary_artist}"
+            )
+            return
+
+        self.allmusic_review = allmusic_data["review"]
+        self.allmusic_rating = allmusic_data["rating"]
+        self.save(update_fields=["allmusic_review", "allmusic_rating"])
+
     def scrape_theaudiodb(self) -> None:
         artist = "Various Artists"
         if self.primary_artist:
@@ -150,6 +213,15 @@ class Album(TimeStampedModel):
 
         Album.objects.filter(pk=self.pk).update(**album_data)
 
+    def scrape_bandcamp(self, force=False) -> None:
+        if not self.bandcamp_id or force:
+            slug = get_bandcamp_slug(self.primary_artist.name, self.name)
+            if not slug:
+                logger.info(f"No bandcamp link for {self}")
+                return
+            self.bandcamp_id = slug
+            self.save(update_fields=["bandcamp_id"])
+
     def fix_metadata(self):
         if (
             not self.musicbrainz_albumartist_id
@@ -198,6 +270,7 @@ class Album(TimeStampedModel):
             ):
                 self.fetch_artwork()
         self.scrape_theaudiodb()
+        self.scrape_allmusic()
 
     def fetch_artwork(self, force=False):
         if not self.cover_image and not force:
@@ -242,7 +315,7 @@ class Album(TimeStampedModel):
     @property
     def allmusic_link(self) -> str:
         if self.allmusic_id:
-            return f"https://www.allmusic.com/artist/{self.allmusic_id}"
+            return f"https://www.allmusic.com/album/{self.allmusic_id}"
         return ""
 
     @property
@@ -263,6 +336,12 @@ class Album(TimeStampedModel):
         album_slug = self.name.lower().replace(" ", "-")
         return f"https://rateyourmusic.com/release/album/{artist_slug}/{album_slug}/"
 
+    @property
+    def bandcamp_link(self):
+        if self.bandcamp_id and self.primary_artist.bandcamp_id:
+            return f"https://{self.primary_artist.bandcamp_id}.bandcamp.com/album/{self.bandcamp_id}"
+        return ""
+
     @property
     def bandcamp_search_link(self):
         artist = self.primary_artist.name.lower()