Merge pull request #1388 from glogiotatidis/jun-update

Upgrade to Boto3 and django-storages==1.9

requirements.txt

@@ -47,9 +47,9 @@ gunicorn==19.9.0 \
 whitenoise==5.0.1 \
     --hash=sha256:62556265ec1011bd87113fb81b7516f52688887b7a010ee899ff1fd18fd22700 \
     --hash=sha256:0f9137f74bd95fa54329ace88d8dc695fbe895369a632e35f7a136e003e41d73
-pytz==2019.2 \
-    --hash=sha256:c894d57500a4cd2d5c71114aaab77dbab5eabd9022308ce5ac9bb93a60a6f0c7 \
-    --hash=sha256:26c0b32e437e54a18161324a2fca3c4b9846b74a8dccddd843113109e1116b32
+pytz==2020.1 \
+    --hash=sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed \
+    --hash=sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048
 newrelic==5.0.2.126 \
     --hash=sha256:9d6d3bf2e125410d485fd91279e1b0f50e8f0f5887284b345aed4ec81db33c0a
 Pygments==2.4.2 \
@@ -104,9 +104,9 @@ factory_boy==2.12.0 \
 cssselect==1.0.3 \
     --hash=sha256:066d8bc5229af09617e24b3ca4d52f1f9092d9e061931f4184cd572885c23204 \
     --hash=sha256:3b5103e8789da9e936a68d993b70df732d06b8bb9a337a05ed4eb52c17ef7206
-flake8==3.7.9 \
-    --hash=sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca \
-    --hash=sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb
+flake8==3.8.2 \
+    --hash=sha256:ccaa799ef9893cebe69fdfefed76865aeaefbb94cb8545617b2298786a4de9a5 \
+    --hash=sha256:c69ac1668e434d37a2d2880b3ca9aafd54b3a10a3ac1ab101d22f29e29cf8634
 pep8==1.7.1 \
     --hash=sha256:b22cfae5db09833bb9bd7c8463b53e1a9c9b39f12e304a8d0bba729c501827ee \
     --hash=sha256:fe249b52e20498e59e0b5c5256aa52ee99fc295b26ec9eaa85776ffdb9fe6374
@@ -163,9 +163,9 @@ boto==2.49.0 \
 django-extensions==2.2.9 \
     --hash=sha256:2f81b618ba4d1b0e58603e25012e5c74f88a4b706e0022a3b21f24f0322a6ce6 \
     --hash=sha256:b19182d101a441fe001c5753553a901e2ef3ff60e8fbbe38881eb4a61fdd17c4
-requests==2.22.0 \
-    --hash=sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4 \
-    --hash=sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31
+requests==2.23.0 \
+    --hash=sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee \
+    --hash=sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6
 tzlocal==1.5.1 \
     --hash=sha256:4ebeb848845ac898da6519b9b31879cf13b6626f7184c496037b818e238f2c4e
 six==1.14.0 \
@@ -214,9 +214,9 @@ APScheduler==3.6.3 \
 funcsigs==1.0.2 \
     --hash=sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca \
     --hash=sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50
-pbr==5.4.1 \
-    --hash=sha256:4aed6c1b1fa5020def0f22aed663d87b81bb3235f112490b07d2643d7a98c5b5 \
-    --hash=sha256:0ca44dc9fd3b04a22297c2a91082d8df2894862e8f4c86a49dac69eae9e85ca0
+pbr==5.4.5 \
+    --hash=sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8 \
+    --hash=sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c
 asn1crypto==1.3.0 \
     --hash=sha256:831d2710d3274c8a74befdddaf9f17fcbf6e350534565074818722d6d615b315 \
     --hash=sha256:5a215cb8dc12f892244e3a113fe05397ee23c5c4ca7a69cd6e69811755efc42d
@@ -419,12 +419,12 @@ Brotli==1.0.7 \
     --hash=sha256:743001bca75f4a6b4454be3510feca46f9d61a0c782a9bc2bc684bdb245e279e \
     --hash=sha256:113f51658e6fe548dce4b3749f6ef6c24de4184ba9c10a909cbee4261c2a5da0 \
     --hash=sha256:0538dc1744fd17c314d2adc409ea7d1b779783b89fd95bcfb0c2acc93a6ea5a7
-django-storages==1.7.1 \
-    --hash=sha256:8e35d2c7baeda5dc6f0b4f9a0fc142d25f9a1bf72b8cebfcbc5db4863abc552d \
-    --hash=sha256:b1a63cd5ea286ee5a9fb45de6c3c5c0ae132d58308d06f1ce9865cfcd5e470a7
-pycodestyle==2.5.0 \
-    --hash=sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56 \
-    --hash=sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c
+pycodestyle==2.6.0 \
+    --hash=sha256:2295e7b2f6b5bd100585ebcb1f616591b652db8a741695b3d8f5d28bdc934367 \
+    --hash=sha256:c58a7d2815e0e8d7972bf1803331fb0152f867bd89adf8a01dfd55085434192e
+django-storages==1.9.1 \
+    --hash=sha256:3103991c2ee8cef8a2ff096709973ffe7106183d211a79f22cf855f33533d924 \
+    --hash=sha256:a59e9923cbce7068792f75344ed7727021ee4ac20f227cf17297d0d03d141e91
 configparser==5.0.0 \
     --hash=sha256:cffc044844040c7ce04e9acd1838b5f2e5fa3170182f6fda4d2ea8b0099dbadd \
     --hash=sha256:2ca44140ee259b5e3d8aaf47c79c36a7ab0d5e94d70bd4105c03ede7a20ea5a1
@@ -524,3 +524,25 @@ pystache==0.5.4 \
 django-ratelimit==2.0.0 \
     --hash=sha256:40dd23dcdda413d2199bb88b4d9151bf66ea19586b2047ada313ddcf77e2959c \
     --hash=sha256:ddb6bd68a7a25fab335a0441671681ce9993167e640a2301a2e0e07ce9dd46fb
+importlib-metadata==1.6.0 \
+    --hash=sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f \
+    --hash=sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e
+zipp==3.1.0 \
+    --hash=sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b \
+    --hash=sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96
+boto3==1.13.19 \
+    --hash=sha256:2e58368d32171f42100353007fb48fe583ea9deafd0ec556aa2bc1ce1582d06d \
+    --hash=sha256:c774003dc13d6de74b5e19d2b84d625da4456e64bd97f44baa1fcf40d808d29a
+s3transfer==0.3.3 \
+    --hash=sha256:2482b4259524933a022d59da830f51bd746db62f047d6eb213f2f8855dcb8a13 \
+    --hash=sha256:921a37e2aefc64145e7b73d50c71bb4f26f46e4c9f414dc648c6245ff92cf7db
+jmespath==0.10.0 \
+    --hash=sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9 \
+    --hash=sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f
+botocore==1.16.19 \
+    --hash=sha256:5cb537e7a4cf2d59a2a8dfbbc8e14ec3bc5b640eb81a1bf3bb0523c0a75e6b1b \
+    --hash=sha256:7b8b1f082665c8670b9aa70143ee527c5d04939fe027a63ac5958359be20ccb0
+docutils==0.15 \
+    --hash=sha256:54a349c622ff31c91cbec43b0b512f113b5b24daf00e2ea530bb1bd9aac14849 \
+    --hash=sha256:ba4584f9107571ced0d2c7f56a5499c696215ba90797849c92d395979da68521 \
+    --hash=sha256:d2ddba74835cb090a1b627d3de4e7835c628d07ee461f7b4480f51af2fe4d448

snippets/base/storage.py

@@ -1,12 +1,8 @@
-import mimetypes
-from datetime import datetime
-
 from django.conf import settings
 from django.core.files.storage import FileSystemStorage
 from django.utils.deconstruct import deconstructible
 
-from boto.utils import ISO8601
-from storages.backends.s3boto import S3BotoStorage
+from storages.backends.s3boto3 import S3Boto3Storage
 
 
 class OverwriteStorage(FileSystemStorage):
@@ -26,42 +22,17 @@ def get_available_name(self, name, max_length=None):
 
 
 @deconstructible
-class S3Storage(S3BotoStorage):
+class S3Storage(S3Boto3Storage):
     cache_control_headers = getattr(settings, 'AWS_CACHE_CONTROL_HEADERS', {})
 
-    def _save(self, name, content):
-        cleaned_name = self._clean_name(name)
-        name = self._normalize_name(cleaned_name)
-        headers = self.headers.copy()
-        _type, _encoding = mimetypes.guess_type(name)
-        content_type = getattr(content, 'content_type', None)
-        content_type = content_type or _type or self.key_class.DefaultContentType
-        encoding = getattr(content, 'content_encoding', None)
-        encoding = encoding or _encoding
-
-        # setting the content_type in the key object is not enough.
-        headers.update({'Content-Type': content_type})
+    def _get_write_parameters(self, name, content):
+        params = super()._get_write_parameters(name, content)
+        encoding = getattr(content, 'content_encoding', params.get('ContentEncoding', None))
+        if encoding:
+            params['ContentEncoding'] = encoding
 
         for filename_start, value in self.cache_control_headers.items():
             if name.startswith(filename_start):
-                headers['Cache-Control'] = value
-
-        if self.gzip and content_type in self.gzip_content_types:
-            content = self._compress_content(content)
-            headers.update({'Content-Encoding': 'gzip'})
-        elif encoding:
-            # If the content already has a particular encoding, set it
-            headers.update({'Content-Encoding': encoding})
-
-        content.name = cleaned_name
-        encoded_name = self._encode_name(name)
-        key = self.bucket.get_key(encoded_name)
-        if not key:
-            key = self.bucket.new_key(encoded_name)
-        if self.preload_metadata:
-            self._entries[encoded_name] = key
-            key.last_modified = datetime.utcnow().strftime(ISO8601)
+                params['CacheControl'] = value
 
-        key.set_metadata('Content-Type', content_type)
-        self._save_content(key, content, headers=headers)
-        return cleaned_name
+        return params

snippets/settings.py

@@ -255,7 +255,8 @@
     AWS_SECRET_ACCESS_KEY = config('AWS_SECRET_ACCESS_KEY')
     AWS_STORAGE_BUCKET_NAME = config('AWS_STORAGE_BUCKET_NAME')
     # Full list of S3 endpoints http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
-    AWS_S3_HOST = config('AWS_S3_HOST')
+    AWS_S3_HOST = f'https://{config("AWS_S3_HOST")}'
+    AWS_S3_REGION_NAME = config('AWS_S3_REGION_NAME', 'us-west-2')
     AWS_CACHE_CONTROL_HEADERS = {
         MEDIA_BUNDLES_ROOT: 'max-age=2592000',  # 1 Month
         MEDIA_ICONS_ROOT: 'max-age=15552000',  # 6 Months