Add text extraction to webpages

2023-11-30 18:19:30 +01:00
parent 33dfbc2c2e
commit 34ecb71c44
4 changed files with 489 additions and 82 deletions
--- a/pyproject.toml
+++ b/pyproject.toml
@ -42,6 +42,7 @@ boto3 = "^1.26.98"
 stream-sqlite = "^0.0.41"
 ipython = "^8.14.0"
 pendulum = "^2.1.2"
+trafilatura = "^1.6.3"

 [tool.poetry.dev-dependencies]
 Werkzeug = "2.0.3"