From 1e17a679d3d411ba982d79b012243c4042b20a51 Mon Sep 17 00:00:00 2001 From: Colin Powell Date: Fri, 24 Mar 2023 14:47:37 -0400 Subject: [PATCH] Allow S3 usage --- poetry.lock | 95 ++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 2 + todos.org | 10 +++-- vrobbler.conf.example | 25 +++++++++--- vrobbler/settings.py | 36 +++++++++++----- 5 files changed, 148 insertions(+), 20 deletions(-) diff --git a/poetry.lock b/poetry.lock index 445978b..888d040 100644 --- a/poetry.lock +++ b/poetry.lock @@ -202,6 +202,38 @@ webencodings = "*" [package.extras] css = ["tinycss2 (>=1.1.0,<1.2)"] +[[package]] +name = "boto3" +version = "1.26.98" +description = "The AWS SDK for Python" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.29.98,<1.30.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.6.0,<0.7.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.29.98" +description = "Low-level, data-driven core of boto 3." +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<1.27" + +[package.extras] +crt = ["awscrt (==0.16.9)"] + [[package]] name = "celery" version = "5.2.7" @@ -541,6 +573,25 @@ category = "main" optional = false python-versions = ">=3.7" +[[package]] +name = "django-storages" +version = "1.13.2" +description = "Support for many storage backends in Django" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +Django = ">=3.2" + +[package.extras] +azure = ["azure-storage-blob (>=12.0.0)"] +boto3 = ["boto3 (>=1.4.4)"] +dropbox = ["dropbox (>=7.2.1)"] +google = ["google-cloud-storage (>=1.27.0)"] +libcloud = ["apache-libcloud"] +sftp = ["paramiko (>=1.10.0)"] + [[package]] name = "django-taggit" version = "2.1.0" @@ -793,6 +844,14 @@ pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib" plugins = ["setuptools"] requirements-deprecated-finder = ["pip-api", "pipreqs"] +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +category = "main" +optional = false +python-versions = ">=3.7" + [[package]] name = "kombu" version = "5.2.4" @@ -1456,6 +1515,20 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9 [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "s3transfer" +version = "0.6.0" +description = "An Amazon S3 Transfer Manager" +category = "main" +optional = false +python-versions = ">= 3.7" + +[package.dependencies] +botocore = ">=1.12.36,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.20.29,<2.0a.0)"] + [[package]] name = "selenium" version = "4.8.2" @@ -1815,7 +1888,7 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "1.1" python-versions = "^3.8" -content-hash = "3f63bf670e22f6aa464abcfc835ba39e7faa27809cc666b35b6ba1cb35313af6" +content-hash = "39b934458b062bcb936d679a51571f6e9375ed27d69dbf996881a9bb8c85daa6" [metadata.files] aiohttp = [ @@ -1987,6 +2060,14 @@ bleach = [ {file = "bleach-6.0.0-py3-none-any.whl", hash = "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"}, {file = "bleach-6.0.0.tar.gz", hash = "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414"}, ] +boto3 = [ + {file = "boto3-1.26.98-py3-none-any.whl", hash = "sha256:983ec9e539431c29b5265e435b91af7c0d77a75809e173427798edb4ede1d69c"}, + {file = "boto3-1.26.98.tar.gz", hash = "sha256:f35a42c6d0130a75e58485efa94383256d9b8c72c3a31ad872807873a8800363"}, +] +botocore = [ + {file = "botocore-1.29.98-py3-none-any.whl", hash = "sha256:ae906c1feb56063a38ffd2280232fa44d634057825470d3beed274925088cb42"}, + {file = "botocore-1.29.98.tar.gz", hash = "sha256:b74283ff71eb4e57edfa5cf6dc36d959b1eec618a2b1e5e781643184857dd1c4"}, +] celery = [ {file = "celery-5.2.7-py3-none-any.whl", hash = "sha256:138420c020cd58d6707e6257b6beda91fd39af7afde5d36c6334d175302c0e14"}, {file = "celery-5.2.7.tar.gz", hash = "sha256:fafbd82934d30f8a004f81e8f7a062e31413a23d444be8ee3326553915958c6d"}, @@ -2290,6 +2371,10 @@ django-simple-history = [ {file = "django-simple-history-3.3.0.tar.gz", hash = "sha256:2313d2d346f15a1e7a92adb3b6696b226f1cd0c1d920869ec40c4c4076614c41"}, {file = "django_simple_history-3.3.0-py3-none-any.whl", hash = "sha256:dc1f98e558a0a1e0b6371c3b8efb85f86e02a6db56e83d0ec198343b7408d00a"}, ] +django-storages = [ + {file = "django-storages-1.13.2.tar.gz", hash = "sha256:cbadd15c909ceb7247d4ffc503f12a9bec36999df8d0bef7c31e57177d512688"}, + {file = "django_storages-1.13.2-py3-none-any.whl", hash = "sha256:31dc5a992520be571908c4c40d55d292660ece3a55b8141462b4e719aa38eab3"}, +] django-taggit = [ {file = "django-taggit-2.1.0.tar.gz", hash = "sha256:a9f41e4ad58efe4b28d86f274728ee87eb98eeae90c9eb4b4efad39e5068184e"}, {file = "django_taggit-2.1.0-py3-none-any.whl", hash = "sha256:61547a23fc99967c9304107414a09e662b459f4163dbbae32e60b8ba40c34d05"}, @@ -2499,6 +2584,10 @@ isort = [ {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, ] +jmespath = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] kombu = [ {file = "kombu-5.2.4-py3-none-any.whl", hash = "sha256:8b213b24293d3417bcf0d2f5537b7f756079e3ea232a8386dcc89a59fd2361a4"}, {file = "kombu-5.2.4.tar.gz", hash = "sha256:37cee3ee725f94ea8bb173eaab7c1760203ea53bbebae226328600f9d2799610"}, @@ -3038,6 +3127,10 @@ rich = [ {file = "rich-13.3.2-py3-none-any.whl", hash = "sha256:a104f37270bf677148d8acb07d33be1569eeee87e2d1beb286a4e9113caf6f2f"}, {file = "rich-13.3.2.tar.gz", hash = "sha256:91954fe80cfb7985727a467ca98a7618e5dd15178cc2da10f553b36a93859001"}, ] +s3transfer = [ + {file = "s3transfer-0.6.0-py3-none-any.whl", hash = "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd"}, + {file = "s3transfer-0.6.0.tar.gz", hash = "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"}, +] selenium = [ {file = "selenium-4.8.2-py3-none-any.whl", hash = "sha256:bd04eb41395605d9b2b65fe587f3fed21431da75512985c52772529e5e210c60"}, {file = "selenium-4.8.2.tar.gz", hash = "sha256:c48372905bffcc3b24bd55ab4683a07ee5e1f30fe918c59558ea5ee44cedf6c3"}, diff --git a/pyproject.toml b/pyproject.toml index 82855d7..9e47876 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,8 @@ celery = "^5.2.7" honcho = "^1.1.0" howlongtobeatpy = "^1.0.5" beautifulsoup4 = "^4.11.2" +django-storages = "^1.13.2" +boto3 = "^1.26.98" [tool.poetry.dev-dependencies] Werkzeug = "2.0.3" diff --git a/todos.org b/todos.org index 52a0a5b..699cfdf 100644 --- a/todos.org +++ b/todos.org @@ -3,10 +3,6 @@ A fun way to keep track of things in the project to fix or improve. * Version 1.0.0 -** TODO [#A] Add django-storage to store files on S3 :settings:improvement: -:LOGBOOK: -CLOCK: [2023-03-24 Fri 10:36]--[2023-03-24 Fri 10:40] => 0:04 -:END: ** TODO Add a "finished_timestamp" so we don't rely on content length :improvement:scrobbling: Essentially, we currently have the timestamp as when the content began @@ -21,6 +17,12 @@ finishes or started. ** TODO Fix bug in Jellyfin scrobbles that spam more scrobbles after completion :scrobbling:videos:bug: ** TODO Fix bug in podcast scrobbling where a second scrobble is created after completion :scrobbling:podcasts:bug: ** TODO Fix KoReader scrobbling to use pages rather than time of last read :scrobbling:books:improvement: +** DONE [#A] Add django-storage to store files on S3 :settings:improvement: +CLOSED: [2023-03-24 Fri 14:46] +:LOGBOOK: +CLOCK: [2023-03-24 Fri 10:47]--[2023-03-24 Fri 14:46] => 3:59 +CLOCK: [2023-03-24 Fri 10:36]--[2023-03-24 Fri 10:40] => 0:04 +:END: ** DONE Fix vrobbler settings not using booleans :settings:bug: CLOSED: [2023-03-24 Fri 10:45] :LOGBOOK: diff --git a/vrobbler.conf.example b/vrobbler.conf.example index 0c1099e..e0131ab 100644 --- a/vrobbler.conf.example +++ b/vrobbler.conf.example @@ -1,11 +1,26 @@ # You can use this file to set environment variables for your local setup # VROBBLER_DEBUG=True -VROBBLER_JSON_LOGGING=True VROBBLER_LOG_LEVEL="DEBUG" +VROBBLER_JSON_LOGGING=True VROBBLER_MEDIA_ROOT = "/media/" -VROBBLER_TMDB_API_KEY = "KEY" -VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=True +VROBBLER_KEEP_DETAILED_SCROBBLE_LOGS=False -VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME" -VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0" +VROBBLER_USE_S3=False +# You may also need to set these in your environment +AWS_S3_ACCESS_KEY_ID="" +AWS_S3_SECRET_ACCESS_KEY="" +AWS_S3_CUSTOM_DOMAIN="https://minio.dev/" + +# API keys +VROBBLER_TMDB_API_KEY = "" +VROBBLER_LASTFM_API_KEY = "" +VROBBLER_LASTFM_SECRET_KEY = "" +VROBBLER_THESPORTSDB_API_KEY="" +VROBBLER_THEAUDIODB_API_KEY="" +VROBBLER_IGDB_CLIENT_ID="" +VROBBLER_IGDB_CLIENT_SECRET="" + +# Storages +# VROBBLER_DATABASE_URL="postgres://USER:PASSWORD@HOST:PORT/NAME" +# VROBBLER_REDIS_URL="redis://:PASS@HOST:6379/0" diff --git a/vrobbler/settings.py b/vrobbler/settings.py index 4ee5f0a..88f3951 100644 --- a/vrobbler/settings.py +++ b/vrobbler/settings.py @@ -101,6 +101,7 @@ INSTALLED_APPS = [ "django.contrib.humanize", "django_filters", "django_extensions", + "storages", "taggit", "rest_framework.authtoken", "encrypted_field", @@ -245,17 +246,32 @@ USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ # -if os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY: - STORAGES = {"default": "storages.backends.s3boto3.S3Boto3Storage"} +from storages.backends import s3boto3 + +if os.getenv("VROBBLER_USE_S3", "False").lower() in TRUTHY: + AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL", "") + AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME", "") + AWS_S3_ACCESS_KEY_ID = os.getenv("AWS_S3_ACCESS_KEY_ID") + AWS_S3_SECRET_ACCESS_KEY = os.getenv("AWS_S3_SECRET_ACCESS_KEY") + + location = "/".join([AWS_S3_ENDPOINT_URL, AWS_STORAGE_BUCKET_NAME]) + print(f"Storing media on S3 at {location}") + + DEFAULT_FILE_storage = "storages.backends.s3boto3.S3Boto3Storage" + STATICFILES_STORAGE = "storages.backends.s3boto3.S3StaticStorage" + STATIC_URL = location + "/static/" + MEDIA_URL = location + "/media/" + +else: + STATIC_ROOT = os.getenv( + "VROBBLER_STATIC_ROOT", os.path.join(PROJECT_ROOT, "static") + ) + MEDIA_ROOT = os.getenv( + "VROBBLER_MEDIA_ROOT", os.path.join(PROJECT_ROOT, "media") + ) + STATIC_URL = os.getenv("VROBBLER_STATIC_URL", "/static/") + MEDIA_URL = os.getenv("VROBBLER_MEDIA_URL", "/media/") -STATIC_URL = os.getenv("VROBBLER_STATIC_URL", "/static/") -STATIC_ROOT = os.getenv( - "VROBBLER_STATIC_ROOT", os.path.join(PROJECT_ROOT, "static") -) -MEDIA_URL = os.getenv("VROBBLER_MEDIA_URL", "/media/") -MEDIA_ROOT = os.getenv( - "VROBBLER_MEDIA_ROOT", os.path.join(PROJECT_ROOT, "media") -) JSON_LOGGING = os.getenv("VROBBLER_JSON_LOGGING", "false").lower() in TRUTHY LOG_TYPE = "json" if JSON_LOGGING else "log"