Allow S3 usage

2023-03-24 14:47:37 -04:00
parent e6914ed079
commit 1e17a679d3
5 changed files with 148 additions and 20 deletions
--- a/poetry.lock
+++ b/poetry.lock
@ -202,6 +202,38 @@ webencodings = "*"
 [package.extras]
 css = ["tinycss2 (>=1.1.0,<1.2)"]

+[[package]]
+name = "boto3"
+version = "1.26.98"
+description = "The AWS SDK for Python"
+category = "main"
+optional = false
+python-versions = ">= 3.7"
+
+[package.dependencies]
+botocore = ">=1.29.98,<1.30.0"
+jmespath = ">=0.7.1,<2.0.0"
+s3transfer = ">=0.6.0,<0.7.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
+
+[[package]]
+name = "botocore"
+version = "1.29.98"
+description = "Low-level, data-driven core of boto 3."
+category = "main"
+optional = false
+python-versions = ">= 3.7"
+
+[package.dependencies]
+jmespath = ">=0.7.1,<2.0.0"
+python-dateutil = ">=2.1,<3.0.0"
+urllib3 = ">=1.25.4,<1.27"
+
+[package.extras]
+crt = ["awscrt (==0.16.9)"]
+
 [[package]]
 name = "celery"
 version = "5.2.7"
@ -541,6 +573,25 @@ category = "main"
 optional = false
 python-versions = ">=3.7"

+[[package]]
+name = "django-storages"
+version = "1.13.2"
+description = "Support for many storage backends in Django"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
+[package.dependencies]
+Django = ">=3.2"
+
+[package.extras]
+azure = ["azure-storage-blob (>=12.0.0)"]
+boto3 = ["boto3 (>=1.4.4)"]
+dropbox = ["dropbox (>=7.2.1)"]
+google = ["google-cloud-storage (>=1.27.0)"]
+libcloud = ["apache-libcloud"]
+sftp = ["paramiko (>=1.10.0)"]
+
 [[package]]
 name = "django-taggit"
 version = "2.1.0"
@ -793,6 +844,14 @@ pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"
 plugins = ["setuptools"]
 requirements-deprecated-finder = ["pip-api", "pipreqs"]

+[[package]]
+name = "jmespath"
+version = "1.0.1"
+description = "JSON Matching Expressions"
+category = "main"
+optional = false
+python-versions = ">=3.7"
+
 [[package]]
 name = "kombu"
 version = "5.2.4"
@ -1456,6 +1515,20 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9
 [package.extras]
 jupyter = ["ipywidgets (>=7.5.1,<9)"]

+[[package]]
+name = "s3transfer"
+version = "0.6.0"
+description = "An Amazon S3 Transfer Manager"
+category = "main"
+optional = false
+python-versions = ">= 3.7"
+
+[package.dependencies]
+botocore = ">=1.12.36,<2.0a.0"
+
+[package.extras]
+crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"]
+
 [[package]]
 name = "selenium"
 version = "4.8.2"
@ -1815,7 +1888,7 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "3f63bf670e22f6aa464abcfc835ba39e7faa27809cc666b35b6ba1cb35313af6"
+content-hash = "39b934458b062bcb936d679a51571f6e9375ed27d69dbf996881a9bb8c85daa6"

 [metadata.files]
 aiohttp = [
@ -1987,6 +2060,14 @@ bleach = [
    {file = "bleach-6.0.0-py3-none-any.whl", hash = "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"},
    {file = "bleach-6.0.0.tar.gz", hash = "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414"},
 ]
+boto3 = [
+    {file = "boto3-1.26.98-py3-none-any.whl", hash = "sha256:983ec9e539431c29b5265e435b91af7c0d77a75809e173427798edb4ede1d69c"},
+    {file = "boto3-1.26.98.tar.gz", hash = "sha256:f35a42c6d0130a75e58485efa94383256d9b8c72c3a31ad872807873a8800363"},
+]
+botocore = [
+    {file = "botocore-1.29.98-py3-none-any.whl", hash = "sha256:ae906c1feb56063a38ffd2280232fa44d634057825470d3beed274925088cb42"},
+    {file = "botocore-1.29.98.tar.gz", hash = "sha256:b74283ff71eb4e57edfa5cf6dc36d959b1eec618a2b1e5e781643184857dd1c4"},
+]
 celery = [
    {file = "celery-5.2.7-py3-none-any.whl", hash = "sha256:138420c020cd58d6707e6257b6beda91fd39af7afde5d36c6334d175302c0e14"},
    {file = "celery-5.2.7.tar.gz", hash = "sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d"},
@ -2290,6 +2371,10 @@ django-simple-history = [
    {file = "django-simple-history-3.3.0.tar.gz", hash = "sha256:2313d2d346f15a1e7a92adb3b6696b226f1cd0c1d920869ec40c4c4076614c41"},
    {file = "django_simple_history-3.3.0-py3-none-any.whl", hash = "sha256:dc1f98e558a0a1e0b6371c3b8efb85f86e02a6db56e83d0ec198343b7408d00a"},
 ]
+django-storages = [
+    {file = "django-storages-1.13.2.tar.gz", hash = "sha256:cbadd15c909ceb7247d4ffc503f12a9bec36999df8d0bef7c31e57177d512688"},
+    {file = "django_storages-1.13.2-py3-none-any.whl", hash = "sha256:31dc5a992520be571908c4c40d55d292660ece3a55b8141462b4e719aa38eab3"},
+]
 django-taggit = [
    {file = "django-taggit-2.1.0.tar.gz", hash = "sha256:a9f41e4ad58efe4b28d86f274728ee87eb98eeae90c9eb4b4efad39e5068184e"},
    {file = "django_taggit-2.1.0-py3-none-any.whl", hash = "sha256:61547a23fc99967c9304107414a09e662b459f4163dbbae32e60b8ba40c34d05"},
@ -2499,6 +2584,10 @@ isort = [
    {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"},
    {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"},
 ]
+jmespath = [
+    {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"},
+    {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"},
+]
 kombu = [
    {file = "kombu-5.2.4-py3-none-any.whl", hash = "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4"},
    {file = "kombu-5.2.4.tar.gz", hash = "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610"},
@ -3038,6 +3127,10 @@ rich = [
    {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"},
    {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"},
 ]
+s3transfer = [
+    {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"},
+    {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"},
+]
 selenium = [
    {file = "selenium-4.8.2-py3-none-any.whl", hash = "sha256:bd04eb41395605d9b2b65fe587f3fed21431da75512985c52772529e5e210c60"},
    {file = "selenium-4.8.2.tar.gz", hash = "sha256:c48372905bffcc3b24bd55ab4683a07ee5e1f30fe918c59558ea5ee44cedf6c3"},
--- a/pyproject.toml
+++ b/pyproject.toml
@ -38,6 +38,8 @@ celery = "^5.2.7"
 honcho = "^1.1.0"
 howlongtobeatpy = "^1.0.5"
 beautifulsoup4 = "^4.11.2"
+django-storages = "^1.13.2"
+boto3 = "^1.26.98"

 [tool.poetry.dev-dependencies]
 Werkzeug = "2.0.3"
--- a/todos.org
+++ b/todos.org
@ -3,10 +3,6 @@
 A fun way to keep track of things in the project to fix or improve.

 * Version 1.0.0
-** TODO [#A] Add django-storage to store files on S3 :settings:improvement:
-:LOGBOOK:
-CLOCK: [2023-03-24 Fri 10:36]--[2023-03-24 Fri 10:40] =>  0:04
-:END:
 ** TODO Add a "finished_timestamp" so we don't rely on content length :improvement:scrobbling:

 Essentially, we currently have the timestamp as when the content began
@ -21,6 +17,12 @@ finishes or started.
 ** TODO Fix bug in Jellyfin scrobbles that spam more scrobbles after completion :scrobbling:videos:bug:
 ** TODO Fix bug in podcast scrobbling where a second scrobble is created after completion :scrobbling:podcasts:bug:
 ** TODO Fix KoReader scrobbling to use pages rather than time of last read :scrobbling:books:improvement:
+** DONE [#A] Add django-storage to store files on S3 :settings:improvement:
+CLOSED: [2023-03-24 Fri 14:46]
+:LOGBOOK:
+CLOCK: [2023-03-24 Fri 10:47]--[2023-03-24 Fri 14:46] =>  3:59
+CLOCK: [2023-03-24 Fri 10:36]--[2023-03-24 Fri 10:40] =>  0:04
+:END:
 ** DONE Fix vrobbler settings not using booleans :settings:bug:
 CLOSED: [2023-03-24 Fri 10:45]
 :LOGBOOK:
--- a/vrobbler.conf.example
+++ b/vrobbler.conf.example
@ -1,11 +1,26 @@
 # You can use this file to set environment variables for your local setup
 #
 VROBBLER_DEBUG=True
-VROBBLER_JSON_LOGGING=True
 VROBBLER_LOG_LEVEL="DEBUG"
+VROBBLER_JSON_LOGGING=True
 VROBBLER_MEDIA_ROOT = "/media/"
-VROBBLER_TMDB_API_KEY = "KEY"
-VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True
+VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=False

-VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"
-VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0"
+VROBBLER_USE_S3=False
+# You may also need to set these in your environment
+AWS_S3_ACCESS_KEY_ID=""
+AWS_S3_SECRET_ACCESS_KEY=""
+AWS_S3_CUSTOM_DOMAIN="https://minio.dev/"
+
+# API keys
+VROBBLER_TMDB_API_KEY = "<key>"
+VROBBLER_LASTFM_API_KEY = "<key>"
+VROBBLER_LASTFM_SECRET_KEY = "<key>"
+VROBBLER_THESPORTSDB_API_KEY="<key>"
+VROBBLER_THEAUDIODB_API_KEY="<key>"
+VROBBLER_IGDB_CLIENT_ID="<id>"
+VROBBLER_IGDB_CLIENT_SECRET="<key>"
+
+# Storages
+# VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME"
+# VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0"
--- a/vrobbler/settings.py
+++ b/vrobbler/settings.py
@ -101,6 +101,7 @@ INSTALLED_APPS = [
    "django.contrib.humanize",
    "django_filters",
    "django_extensions",
+    "storages",
    "taggit",
    "rest_framework.authtoken",
    "encrypted_field",
@ -245,17 +246,32 @@ USE_TZ = True
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.1/howto/static-files/
 #
-if os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY:
-    STORAGES = {"default": "storages.backends.s3boto3.S3Boto3Storage"}
+from storages.backends import s3boto3
+
+if os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY:
+    AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", "")
+    AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "")
+    AWS_S3_ACCESS_KEY_ID = os.getenv("AWS_S3_ACCESS_KEY_ID")
+    AWS_S3_SECRET_ACCESS_KEY = os.getenv("AWS_S3_SECRET_ACCESS_KEY")
+
+    location = "/".join([AWS_S3_ENDPOINT_URL, AWS_STORAGE_BUCKET_NAME])
+    print(f"Storing media on S3 at {location}")
+
+    DEFAULT_FILE_storage = "storages.backends.s3boto3.S3Boto3Storage"
+    STATICFILES_STORAGE = "storages.backends.s3boto3.S3StaticStorage"
+    STATIC_URL = location + "/static/"
+    MEDIA_URL = location + "/media/"
+
+else:
+    STATIC_ROOT = os.getenv(
+        "VROBBLER_STATIC_ROOT", os.path.join(PROJECT_ROOT, "static")
+    )
+    MEDIA_ROOT = os.getenv(
+        "VROBBLER_MEDIA_ROOT", os.path.join(PROJECT_ROOT, "media")
+    )
+    STATIC_URL = os.getenv("VROBBLER_STATIC_URL", "/static/")
+    MEDIA_URL = os.getenv("VROBBLER_MEDIA_URL", "/media/")

-STATIC_URL = os.getenv("VROBBLER_STATIC_URL", "/static/")
-STATIC_ROOT = os.getenv(
-    "VROBBLER_STATIC_ROOT", os.path.join(PROJECT_ROOT, "static")
-)
-MEDIA_URL = os.getenv("VROBBLER_MEDIA_URL", "/media/")
-MEDIA_ROOT = os.getenv(
-    "VROBBLER_MEDIA_ROOT", os.path.join(PROJECT_ROOT, "media")
-)

 JSON_LOGGING = os.getenv("VROBBLER_JSON_LOGGING", "false").lower() in TRUTHY
 LOG_TYPE = "json" if JSON_LOGGING else "log"