Switch to boto3's generate_presigned_post method

README.md

@@ -2,8 +2,6 @@
 
 A lightweight file upload input for Django and Amazon S3.
 
-_less than 200 lines and no dependencies_
-
 [![PyPi Version](https://img.shields.io/pypi/v/django-s3file.svg)](https://pypi.python.org/pypi/django-s3file/)
 [![Build Status](https://travis-ci.org/codingjoe/django-s3file.svg?branch=master)](https://travis-ci.org/codingjoe/django-s3file)
 [![Test Coverage](https://coveralls.io/repos/codingjoe/django-s3file/badge.svg?branch=master)](https://coveralls.io/r/codingjoe/django-s3file)
@@ -48,8 +46,7 @@ By default S3File will replace Django's `FileField` widget,
 but you can also specify the widget manually and pass custom attributes.
 
 The `FileField`'s widget is only than automatically replaced when the
-`AWS_SECRET_ACCESS_KEY` setting is set. This setting is required
-by `django-storages` to setup the Boto3 storage.
+`DEFAULT_FILE_STORAGE` setting is set to `django-storages`' `S3Boto3Storage`.
 
 ### Simple integrations
 

s3file/apps.py

@@ -1,16 +1,20 @@
 from django.apps import AppConfig
-from django.conf import settings
+
+try:
+    from storages.backends.s3boto3 import S3Boto3Storage
+except ImportError:
+    from storages.backends.s3boto import S3BotoStorage as S3BotoStorage
 
 
 class S3FileConfig(AppConfig):
     name = 's3file'
-    verbose_name = "S3File"
+    verbose_name = 'S3File'
 
     def ready(self):
         from django.forms import FileField
+        from django.core.files.storage import default_storage
 
-        if hasattr(settings, 'AWS_SECRET_ACCESS_KEY') \
-                and settings.AWS_SECRET_ACCESS_KEY:
+        if isinstance(default_storage, S3Boto3Storage):
             from .forms import S3FileInput
 
             FileField.widget = S3FileInput

s3file/forms.py

@@ -1,17 +1,11 @@
-import hashlib
-import hmac
-import json
 import logging
 import os
 import uuid
-from base64 import b64encode
 
 from django.conf import settings
+from django.core.files.storage import default_storage
 from django.forms.widgets import ClearableFileInput
-from django.utils.encoding import force_text
 from django.utils.functional import cached_property
-from django.utils.six import binary_type
-from django.utils.timezone import datetime, timedelta
 
 logger = logging.getLogger('s3file')
 
@@ -23,49 +17,41 @@ class S3FileInput(ClearableFileInput):
     mime_type = None
 
     def __init__(self, attrs=None):
-        self.expires = timedelta(seconds=settings.SESSION_COOKIE_AGE)
-        self.access_key = settings.AWS_ACCESS_KEY_ID
-        self.secret_access_key = settings.AWS_SECRET_ACCESS_KEY
-        self.bucket_name = settings.AWS_STORAGE_BUCKET_NAME
+        self.expires = settings.SESSION_COOKIE_AGE
         self.upload_path = getattr(settings, 'S3FILE_UPLOAD_PATH', os.path.join('tmp', 's3file'))
         super(S3FileInput, self).__init__(attrs=attrs)
         try:
             self.mime_type = self.attrs['accept']
         except KeyError:
             pass
 
-    def get_expiration_date(self):
-        expiration_date = datetime.utcnow() + self.expires
-        return expiration_date.strftime('%Y-%m-%dT%H:%M:%S.000Z')
+    @property
+    def bucket_name(self):
+        return default_storage.bucket.name
+
+    @property
+    def client(self):
+        return default_storage.connection.meta.client
 
     def build_attrs(self, *args, **kwargs):
         attrs = super(S3FileInput, self).build_attrs(*args, **kwargs)
+        response = self.client.generate_presigned_post(
+            self.bucket_name, os.path.join(self.upload_folder, '${filename}'),
+            Conditions=self.get_conditions(),
+            ExpiresIn=self.expires,
+        )
         defaults = {
-            'data-policy': force_text(self.get_policy()),
-            'data-signature': self.get_signature(),
-            'data-key': self.upload_folder,
-            'data-s3-url': 'https://s3.amazonaws.com/%s' % self.bucket_name,
-            'data-AWSAccessKeyId': self.access_key,
+            'data-fields-%s' % key: value
+            for key, value in response['fields'].items()
         }
+        defaults['data-url'] = response['url']
         defaults.update(attrs)
         try:
             defaults['class'] += ' s3file'
         except KeyError:
             defaults['class'] = 's3file'
         return defaults
 
-    def get_secret_access_key(self):
-        return binary_type(self.secret_access_key.encode('utf-8'))
-
-    def get_policy(self):
-        policy = {
-            "expiration": self.get_expiration_date(),
-            "conditions": self.get_conditions(),
-        }
-        policy_json = json.dumps(policy)
-        policy_json = policy_json.replace('\n', '').replace('\r', '')
-        return b64encode(binary_type(policy_json.encode('utf-8')))
-
     def get_conditions(self):
         conditions = [
             {"bucket": self.bucket_name},
@@ -90,21 +76,6 @@ def upload_folder(self):
             uuid.uuid4().hex,
         )
 
-    def get_signature(self):
-        """
-        Return S3 upload signature.
-
-        :rtype: dict
-        """
-        policy_object = self.get_policy()
-        signature = hmac.new(
-            self.get_secret_access_key(),
-            policy_object,
-            hashlib.sha1
-        ).digest()
-
-        return force_text(b64encode(signature))
-
     class Media:
         js = (
             's3file/js/s3file.js',

s3file/middleware.py

@@ -2,8 +2,13 @@
 
 from django.core.files.storage import default_storage
 
+try:
+    from django.utils.deprecation import MiddlewareMixin
+except ImportError:
+    MiddlewareMixin = object
 
-class S3FileMiddleware(object):
+
+class S3FileMiddleware(MiddlewareMixin):
     @staticmethod
     def get_files_from_storage(paths):
         """Return S3 file where the name does not include the path."""
@@ -12,11 +17,8 @@ def get_files_from_storage(paths):
             f.name = os.path.basename(path)
             yield f
 
-    def __call__(self, request):
+    def process_request(self, request):
         file_fields = request.POST.getlist('s3file', [])
         for field_name in file_fields:
             paths = request.POST.getlist(field_name, [])
             request.FILES.setlist(field_name, list(self.get_files_from_storage(paths)))
-
-    def process_request(self, request):
-        self.__call__(request)

s3file/static/s3file/js/s3file.js

@@ -35,19 +35,19 @@
   }
 
   function uploadFiles (e, fileInput, name) {
-    const url = fileInput.getAttribute('data-s3-url')
+    const url = fileInput.getAttribute('data-url')
     const form = e.target
-    const policy = fileInput.getAttribute('data-policy')
-    const signature = fileInput.getAttribute('data-signature')
-    const AWSAccessKeyId = fileInput.getAttribute('data-AWSAccessKeyId')
     const promises = Array.from(fileInput.files).map((file) => {
       const s3Form = new window.FormData()
-      s3Form.append('policy', policy)
-      s3Form.append('signature', signature)
-      s3Form.append('AWSAccessKeyId', AWSAccessKeyId)
+      Array.from(fileInput.attributes).forEach(attr => {
+        let name = attr.name
+        if (name.startsWith('data-fields')) {
+          name = name.replace('data-fields-', '')
+          s3Form.append(name, attr.value)
+        }
+      })
       s3Form.append('success_action_status', '201')
       s3Form.append('Content-Type', file.type)
-      s3Form.append('key', fileInput.getAttribute('data-key') + '/' + file.name)
       s3Form.append('file', file)
       return request('POST', url, s3Form)
     })

setup.py

@@ -23,5 +23,7 @@
     ],
     packages=['s3file'],
     include_package_data=True,
-    install_requires=[],
+    install_requires=[
+        'django-storages',
+    ],
 )

tests/test_forms.py

@@ -1,6 +1,3 @@
-import base64
-import datetime
-import json
 from contextlib import contextmanager
 
 import pytest
@@ -10,7 +7,7 @@
 from selenium.webdriver.support.wait import WebDriverWait
 
 from s3file.forms import S3FileInput
-from tests.testapp.forms import ClearableUploadForm, UploadForm
+from tests.testapp.forms import UploadForm
 
 try:
     from django.urls import reverse
@@ -36,21 +33,8 @@ def url(self):
     @pytest.fixture
     def freeze(self, monkeypatch):
         """Freeze datetime and UUID."""
-        monkeypatch.setattr('s3file.forms.S3FileInput.get_expiration_date',
-                            lambda _: '1988-11-19T10:10:00.000Z')
         monkeypatch.setattr('s3file.forms.S3FileInput.upload_folder', 'tmp')
 
-    def test_get_expiration_date(self, settings):
-        DATE_FORMAT = '%Y-%m-%dT%H:%M:%S.000Z'
-        assert S3FileInput().get_expiration_date()[-1] == 'Z', 'is UTC date'
-        date_1 = datetime.datetime.strptime(S3FileInput().get_expiration_date(), DATE_FORMAT)
-        settings.SESSION_COOKIE_AGE = 60 * 60 * 24 * 7 * 1
-        date_2 = datetime.datetime.strptime(S3FileInput().get_expiration_date(), DATE_FORMAT)
-        assert date_2 < date_1, S3FileInput().get_expiration_date()
-
-    def test_get_expiration_date_freeze(self, freeze):
-        assert S3FileInput().get_expiration_date() == '1988-11-19T10:10:00.000Z'
-
     def test_value_from_datadict(self, client, upload_file):
         with open(upload_file) as f:
             uploaded_file = default_storage.save('test.jpg', f)
@@ -83,49 +67,32 @@ def test_initial_fallback(self, filemodel):
         assert form.cleaned_data['file'] == filemodel.file
 
     def test_clear(self, filemodel):
-        form = ClearableUploadForm(data={'file-clear': '1'}, instance=filemodel)
+        form = UploadForm(data={'file-clear': '1'}, instance=filemodel)
         assert form.is_valid()
         assert not form.cleaned_data['file']
 
     def test_build_attr(self):
         assert set(S3FileInput().build_attrs({}).keys()) == {
             'class',
-            'data-AWSAccessKeyId',
-            'data-s3-url',
-            'data-key',
-            'data-policy',
-            'data-signature',
+            'data-url',
+            'data-fields-x-amz-algorithm',
+            'data-fields-x-amz-date',
+            'data-fields-x-amz-signature',
+            'data-fields-x-amz-credential',
+            'data-fields-policy',
+            'data-fields-key',
         }
         assert S3FileInput().build_attrs({})['class'] == 's3file'
         assert S3FileInput().build_attrs({'class': 'my-class'})['class'] == 'my-class s3file'
 
-    def test_get_policy(self, freeze):
-        base64_policy = S3FileInput().get_policy()
-        policy = json.loads(base64.b64decode(base64_policy).decode('utf-8'))
-        assert policy == {
-            'expiration': '1988-11-19T10:10:00.000Z',
-            'conditions': [
-                {'bucket': 'test-bucket'},
-                ['starts-with', '$key', 'tmp'],
-                {'success_action_status': '201'},
-                ['starts-with', '$Content-Type', ''],
-            ],
-        }
-
-    def test_get_signature(self, freeze):
-        assert S3FileInput().get_signature() in [
-            'jdvyRM/sS2frI9oSe6vIXGFswqg=',
-            'D3W1aKcI1VkzcFHrvSQbGdqnmPo=',
-        ]
-
     def test_get_conditions(self, freeze):
         conditions = S3FileInput().get_conditions()
         assert all(condition in conditions for condition in [
             {"bucket": 'test-bucket'},
             {"success_action_status": "201"},
             ['starts-with', '$key', 'tmp'],
             ["starts-with", "$Content-Type", ""]
-        ])
+        ]), conditions
 
     def test_accept(self):
         widget = S3FileInput()
@@ -153,8 +120,6 @@ def test_no_js_error(self, driver, live_server):
     def test_file_insert(self, request, driver, live_server, upload_file, freeze):
         driver.get(live_server + self.url)
         file_input = driver.find_element_by_xpath('//input[@type=\'file\']')
-        driver.execute_script('arguments[0].setAttribute("data-s3-url", arguments[1])',
-                              file_input, live_server + reverse('s3mock'))
         file_input.send_keys(upload_file)
         assert file_input.get_attribute('name') == 'file'
         with wait_for_page_load(driver, timeout=10):

tests/testapp/forms.py

@@ -1,4 +1,5 @@
 from django import forms
+from s3file.forms import S3FileInput
 
 from .models import FileModel
 
@@ -7,9 +8,6 @@ class UploadForm(forms.ModelForm):
     class Meta:
         model = FileModel
         fields = ('file',)
-
-
-class ClearableUploadForm(forms.ModelForm):
-    class Meta:
-        model = FileModel
-        fields = ('file',)
+        widgets = {
+            'file': S3FileInput
+        }

tests/testapp/settings.py

@@ -16,10 +16,13 @@
     'django.contrib.sessions',
     'django.contrib.staticfiles',
 
+    'storages',
     's3file',
     'tests.testapp',
 )
 
+DEFAULT_FILE_STORAGE = 'tests.testapp.storages.DummyS3Boto3Storage'
+
 MIDDLEWARE = MIDDLEWARE_CLASSES = (
     'django.contrib.sessions.middleware.SessionMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
@@ -47,3 +50,5 @@
 AWS_ACCESS_KEY_ID = 'testaccessid'
 AWS_SECRET_ACCESS_KEY = 'supersecretkey'
 AWS_STORAGE_BUCKET_NAME = 'test-bucket'
+AWS_S3_REGION_NAME = 'eu-central-1'
+AWS_S3_SIGNATURE_VERSION = 's3v4'

tests/testapp/storages.py

@@ -0,0 +1,29 @@
+from django.core.files.storage import FileSystemStorage
+
+try:
+    from django.urls import reverse
+except ImportError:
+    # Django 1.8 support
+    from django.core.urlresolvers import reverse
+
+
+class DummyS3Boto3Storage(FileSystemStorage):
+    class connection:
+        class meta:
+            class client:
+                @staticmethod
+                def generate_presigned_post(*args, **kargs):
+                    return {
+                        'url': reverse('s3mock'),
+                        'fields': {
+                            'x-amz-algorithm': 'AWS4-HMAC-SHA256',
+                            'x-amz-date': '20170908T111600Z',
+                            'x-amz-signature': 'asdf',
+                            'x-amz-credential': 'testaccessid',
+                            'policy': 'asdf',
+                            'key': 'tmp/${filename}',
+                        },
+                    }
+
+    class bucket:
+        name = 'test-bucket'