[webpages] Add more robust scraping

2026-06-23 23:04:30 -04:00
parent 327ba94c63
commit 2283a6c640
4 changed files with 106 additions and 9 deletions
--- a/pyproject.toml
+++ b/pyproject.toml
@ -11,6 +11,8 @@ django-extensions = "^3.1.5"
 python-dateutil = "^2.8.2"
 python-dotenv = ">=0.20.0,<2"
 python-json-logger = "^2.0.2"
+cloudscraper = "^1.2.71"
+curl-cffi = "^0.15.0"
 colorlog = "^6.6.0"
 httpx = "<=0.27.2"
 djangorestframework = "^3.13.1"