Merge pull request #16 from ImperialCollegeLondon/feature/storage

Adding support for django-storages

.coveragerc

@@ -1,3 +1,7 @@
 [run]
-omit = django_drf_filepond/__init__.py
-       django_drf_filepond/apps.py
+omit = 
+    django_drf_filepond/__init__.py
+    django_drf_filepond/apps.py
+
+[report]
+show_missing = True

django_drf_filepond/apps.py

@@ -1,32 +1,57 @@
 from django.apps import AppConfig
 
+import importlib
 import os
 import logging
 import django_drf_filepond.drf_filepond_settings as local_settings
+from django.core.exceptions import ImproperlyConfigured
 
 LOG = logging.getLogger(__name__)
 
+
 class DjangoDrfFilepondConfig(AppConfig):
     name = 'django_drf_filepond'
     verbose_name = 'FilePond Server-side API'
-    
+
     def ready(self):
-        upload_tmp = getattr(local_settings, 'UPLOAD_TMP',
-                             os.path.join(
-                                 local_settings.BASE_DIR,'filepond_uploads'))
+        upload_tmp = getattr(
+            local_settings, 'UPLOAD_TMP',
+            os.path.join(local_settings.BASE_DIR, 'filepond_uploads'))
+
         LOG.debug('Upload temp directory from top level settings: <%s>'
                   % (upload_tmp))
-        
+
         # Check if the temporary file directory is available and if not
         # create it.
         if not os.path.exists(local_settings.UPLOAD_TMP):
             LOG.warning('Filepond app init: Creating temporary file '
-                     'upload directory <%s>' % local_settings.UPLOAD_TMP)
+                        'upload directory <%s>' % local_settings.UPLOAD_TMP)
             os.makedirs(local_settings.UPLOAD_TMP, mode=0o700)
         else:
             LOG.debug('Filepond app init: Temporary file upload '
                       'directory already exists')
 
+        # See if we're using a local file store or django-storages
+        # If the latter, we create an instance of the storage class
+        # to make sure that it's available and dependencies are installed
+        storage_class = getattr(local_settings, 'STORAGES_BACKEND', None)
+        if storage_class:
+            LOG.info('Using django-storages with backend [%s]'
+                     % storage_class)
+            (modname, clname) = storage_class.rsplit('.', 1)
+            # Either the module import or the getattr to instantiate the
+            # class will throw an exception if there's a problem
+            # creating storage backend instance due to missing configuration
+            # or dependencies.
+            mod = importlib.import_module(modname)
+            getattr(mod, clname)()
+            LOG.info('Storage backend [%s] is available...' % storage_class)
+        else:
+            LOG.info('App init: no django-storages backend configured, '
+                     'using default (local) storage backend if set, '
+                     'otherwise you need to manage file storage '
+                     'independently of this app.')
+
         file_store = getattr(local_settings, 'FILE_STORE_PATH', None)
         if file_store:
             if not os.path.exists(file_store):
@@ -35,4 +60,10 @@ def ready(self):
                 os.makedirs(file_store, mode=0o700)
             else:
                 LOG.debug('Filepond app init: File store path already '
-                          'exists')
+                          'exists')
+        else:
+            if not storage_class:
+                raise ImproperlyConfigured(
+                    'You are using local file storage so you must set the '
+                    'base file storage path using %sFILE_STORE_PATH'
+                    % local_settings._app_prefix)

django_drf_filepond/drf_filepond_settings.py

@@ -5,8 +5,8 @@
 http://blog.muhuk.com/2010/01/26/developing-reusable-django-apps-app-settings.html#.W_Rkh-mYQ6U
 
 ******** Settings in this file are not currently being used for core library
-******** use due to an issue with dynamically setting up the storage  
-******** location in the FileSystemStorage class. For now, you must set 
+******** use due to an issue with dynamically setting up the storage
+******** location in the FileSystemStorage class. For now, you must set
 ******** DJANGO_DRF_FILEPOND_UPLOAD_TMP in your top level app settings.
 ******** THIS IS CURRENTLY USED ONLY FOR INTEGRATION TESTS
 '''
@@ -17,23 +17,36 @@
 _app_prefix = 'DJANGO_DRF_FILEPOND_'
 
 # BASE_DIR is assumed to be present in the core project settings. However,
-# in case it isn't, we check here and set a local BASE_DIR to the 
+# in case it isn't, we check here and set a local BASE_DIR to the
 # installed app base directory to use as an alternative.
 BASE_DIR = os.path.dirname(django_drf_filepond.__file__)
 if hasattr(settings, 'BASE_DIR'):
     BASE_DIR = settings.BASE_DIR
 
-# The location where uploaded files are temporarily stored. At present, 
+# The location where uploaded files are temporarily stored. At present,
 # this must be a subdirectory of settings.BASE_DIR
 UPLOAD_TMP = getattr(settings, _app_prefix+'UPLOAD_TMP',
-                     os.path.join(BASE_DIR,'filepond_uploads'))
+                     os.path.join(BASE_DIR, 'filepond_uploads'))
 
-# Setting to control whether the temporary directories created for file 
+# Setting to control whether the temporary directories created for file
 # uploads are removed when an uploaded file is deleted
-DELETE_UPLOAD_TMP_DIRS = getattr(settings, 
+DELETE_UPLOAD_TMP_DIRS = getattr(settings,
                                  _app_prefix+'DELETE_UPLOAD_TMP_DIRS', True)
 
-# The file storage location used by the top-level application. This needs to 
-# be set if the load endpoint is going to be used to access files that have 
+# Specifies the django-storages backend to be used. See the list at:
+# https://django-storages.readthedocs.io
+# If this is not set, then the default local filesystem backend is used.
+# If you set this parameter, you also need to set the relevant parameters
+# for your chosen backend as described in the django-storages documentation.
+STORAGES_BACKEND = getattr(settings, _app_prefix+'STORAGES_BACKEND', None)
+
+# The file storage location used by the top-level application. This needs to
+# be set if the load endpoint is going to be used to access files that have
 # been permanently stored after being uploaded as TemporaryUpload objects.
-FILE_STORE_PATH = getattr(settings, _app_prefix+'FILE_STORE_PATH', None)
+# If you're using django-storages, this path is the base path to be used
+# on the backend storage.
+# If STORAGES_BACKEND is provided, this MUST be set
+# If you're not using a STORAGES_BACKEND and this is NOT set, you can't use
+# django-drf-filepond's file management and the api.store_upload function
+# will not be usable - you will need to manage file storage in your code.
+FILE_STORE_PATH = getattr(settings, _app_prefix+'FILE_STORE_PATH', None)

django_drf_filepond/exceptions.py

@@ -0,0 +1,12 @@
+class ConfigurationError(Exception):
+    '''
+    Raised when a problem occurs with the configuration of the library.
+    '''
+    pass
+
+
+class APIError(Exception):
+    '''
+    Raised when a problem occurs in the API functions.
+    '''
+    pass

django_drf_filepond/models.py

@@ -14,9 +14,9 @@
 import django_drf_filepond.drf_filepond_settings as local_settings
 
 
-FILEPOND_UPLOAD_TMP = getattr(local_settings, 'UPLOAD_TMP',
-                              os.path.join(
-                                  local_settings.BASE_DIR,'filepond_uploads'))
+FILEPOND_UPLOAD_TMP = getattr(
+    local_settings, 'UPLOAD_TMP',
+    os.path.join(local_settings.BASE_DIR, 'filepond_uploads'))
 
 
 @deconstructible
@@ -48,62 +48,67 @@ def __init__(self, **kwargs):
 def get_upload_path(instance, filename):
     return os.path.join(instance.upload_id, filename)
 
+
 class TemporaryUpload(models.Model):
-    
+
     FILE_DATA = 'F'
     URL = 'U'
     UPLOAD_TYPE_CHOICES = (
         (FILE_DATA, 'Uploaded file data'),
         (URL, 'Remote file URL'),
     )
-    
-    # The unique ID returned to the client and the name of the temporary 
+
+    # The unique ID returned to the client and the name of the temporary
     # directory created to hold file data
-    upload_id = models.CharField(primary_key=True, max_length=22, 
-                               validators=[MinLengthValidator(22)])
+    upload_id = models.CharField(primary_key=True, max_length=22,
+                                 validators=[MinLengthValidator(22)])
     # The unique ID used to store the file itself
-    file_id = models.CharField(max_length=22, 
+    file_id = models.CharField(max_length=22,
                                validators=[MinLengthValidator(22)])
     file = models.FileField(storage=storage, upload_to=get_upload_path)
     upload_name = models.CharField(max_length=512)
     uploaded = models.DateTimeField(auto_now_add=True)
-    upload_type = models.CharField(max_length = 1, 
+    upload_type = models.CharField(max_length=1,
                                    choices=UPLOAD_TYPE_CHOICES)
-    
+
     def get_file_path(self):
         return self.file.path
 
+
 class StoredUpload(models.Model):
-        
-    # The unique upload ID assigned to this file when it was originally 
-    # uploaded (or retrieved from a remote URL) 
-    upload_id = models.CharField(primary_key=True, max_length=22, 
-                               validators=[MinLengthValidator(22)])
-    # The file name and path (relative to the base file store directory 
-    # as set by DJANGO_DRF_FILEPOND_FILE_STORE_PATH).
+
+    # The unique upload ID assigned to this file when it was originally
+    # uploaded (or retrieved from a remote URL)
+    upload_id = models.CharField(primary_key=True, max_length=22,
+                                 validators=[MinLengthValidator(22)])
+    # The file name and path (relative to the base file store directory
+    # as set by DJANGO_DRF_FILEPOND_FILE_STORE_PATH).
     file_path = models.CharField(max_length=2048)
     uploaded = models.DateTimeField()
     stored = models.DateTimeField(auto_now_add=True)
-    
+
     def get_absolute_file_path(self):
-        return os.path.join(local_settings.FILE_STORE_PATH, self.file_path)
-
-# When a TemporaryUpload record is deleted, we need to delete the 
+        fsp = local_settings.FILE_STORE_PATH
+        if not fsp:
+            fsp = ''
+        return os.path.join(fsp, self.file_path)
+
+
+# When a TemporaryUpload record is deleted, we need to delete the
 # corresponding file from the filesystem by catching the post_delete signal.
 @receiver(post_delete, sender=TemporaryUpload)
 def delete_temp_upload_file(sender, instance, **kwargs):
     # Check that the file parameter for the instance is not None
     # and that the file exists and is not a directory! Then we can delete it
     LOG.debug('*** post_delete signal handler called. Deleting file.')
     if instance.file:
-        if (os.path.exists(instance.file.path) and 
-            os.path.isfile(instance.file.path)):
+        if (os.path.exists(instance.file.path) and
+                os.path.isfile(instance.file.path)):
             os.remove(instance.file.path)
-    
+
     if local_settings.DELETE_UPLOAD_TMP_DIRS:
         file_dir = os.path.join(storage.location, instance.upload_id)
         if(os.path.exists(file_dir) and os.path.isdir(file_dir)):
             os.rmdir(file_dir)
             LOG.debug('*** post_delete signal handler called. Deleting temp '
                       'dir that contained file.')
-

django_drf_filepond/parsers.py

@@ -8,11 +8,12 @@
 '''
 from rest_framework.parsers import BaseParser
 
-# This plaintext parser is taken from the example in the 
-# django rest framework docs since this provides exactly what we 
+
+# This plaintext parser is taken from the example in the
+# django rest framework docs since this provides exactly what we
 # require but doesn't seem to be included in the core DRF API.
 # See: https://www.django-rest-framework.org/api-guide/parsers/#example
-# This will make the data from the body of the request available 
+# This will make the data from the body of the request available
 # in request.data
 class PlainTextParser(BaseParser):
     """
@@ -24,4 +25,4 @@ def parse(self, stream, media_type=None, parser_context=None):
         """
         Simply return a string representing the body of the request.
         """
-        return stream.read()
+        return stream.read()

django_drf_filepond/renderers.py

@@ -15,11 +15,12 @@
 
 LOG = logging.getLogger(__name__)
 
-# This plaintext renderer is taken from the example in the 
-# django rest framework docs since this provides exactly what we 
-# require but doesn't seem to be included in the core DRF API.
-# See: https://www.django-rest-framework.org/api-guide/renderers/#custom-renderers
-# This renderer avoids the issue with the standard JSONRenderer that 
+
+# This plaintext renderer is taken from the example in the
+# django rest framework docs since this provides exactly what we
+# require but doesn't seem to be included in the core DRF API. See:
+# https://www.django-rest-framework.org/api-guide/renderers/#custom-renderers
+# This renderer avoids the issue with the standard JSONRenderer that
 # results in raw text responses being wrapped in quotes.
 class PlainTextRenderer(BaseRenderer):
     '''
@@ -33,10 +34,10 @@ def render(self, data, media_type=None, renderer_context=None):
         Encode the raw data - default charset is UTF-8.
         '''
         LOG.debug('Data is <%s>' % data)
-        
+
         if data:
             if type(data) in [dict, OrderedDict]:
                 return json.dumps(data)
             else:
                 return data.encode(self.charset)
-        return data
+        return data

django_drf_filepond/storage_utils.py

@@ -0,0 +1,32 @@
+import importlib
+import logging
+
+
+LOG = logging.getLogger(__name__)
+
+
+def _get_storage_backend(fq_classname):
+    """
+    Load the specified django-storages storage backend class. This is called
+    regardless of whether a beckend is specified so if fq_classname is not
+    set, we just return None.
+
+    fq_classname is a string specifying the fully-qualified class name of
+    the django-storages backend to use, e.g.
+        'storages.backends.sftpstorage.SFTPStorage'
+    """
+    LOG.debug('Running _get_storage_backend with fq_classname [%s]'
+              % fq_classname)
+
+    if not fq_classname:
+        return None
+
+    (modname, clname) = fq_classname.rsplit('.', 1)
+    # A test import of the backend storage class should have been undertaken
+    # at app startup in django_drf_filepond.apps.ready so any failure
+    # importing the backend should have been picked up then.
+    mod = importlib.import_module(modname)
+    storage_backend = getattr(mod, clname)()
+    LOG.info('Storage backend instance [%s] created...' % fq_classname)
+
+    return storage_backend

django_drf_filepond/urls.py

@@ -1,7 +1,7 @@
 """FilePond server-side URL configuration
 
 Based on the server-side configuration details
-provided at: 
+provided at:
 https://pqina.nl/filepond/docs/patterns/api/server/#configuration
 """
 from django.conf.urls import url