diff --git a/vrobbler/apps/webpages/models.py b/vrobbler/apps/webpages/models.py index fd580f1..7733b7b 100644 --- a/vrobbler/apps/webpages/models.py +++ b/vrobbler/apps/webpages/models.py @@ -59,11 +59,9 @@ class WebPage(ScrobblableMixin): def _raw_domain(self): self.url.split("//")[-1].split("/")[0] - def _update_extract_from_web(self, force=True): - headers = { - "headers": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:51.0) Gecko/20100101 Firefox/51.0" - } - raw_text = requests.get(self.url, headers=headers).text + def _update_extract_from_web(self, raw_text: str = "", force=True): + if not raw_text: + raw_text = requests.get(self.url, headers=headers).text if not self.extract or force: self.extract = trafilatura.extract(raw_text) self.save(update_fields=["extract"]) @@ -95,34 +93,38 @@ class WebPage(ScrobblableMixin): ) def clean_title(self, title: str, save=True): - if len(title.split('|')) > 1: - if "The Quietus" in title: - title = title.split('|')[-0] - else: - title = title.split('|')[0] - if len(title.split('–')) > 1: - title = title.split('–')[0] - if len(title.split(' - ')) > 1: - title = title.split(' - ')[0] + if len(title.split("|")) > 1: + title = title.split("|")[0] + if len(title.split("–")) > 1: + title = title.split("–")[0] + if len(title.split(" - ")) > 1: + title = title.split(" - ")[0] self.title = title.strip() + if save: self.save(update_fields=["title"]) - - - def _update_domain_from_url(self): + def _update_domain_from_url(self, save=False): domain = self.url.split("//")[-1].split("/")[0].split("www.")[-1] self.domain, created = Domain.objects.get_or_create(root=domain) - def _update_data_from_web(self, force=True): - headers = { - "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:51.0) Gecko/20100101 Firefox/51.0" - } - raw_text = requests.get(self.url, headers=headers).text - if not self.title or force: - self.set_title(raw_text[ - raw_text.find("