[#1261] add filesize limit, make storage folder backwards compatable,…

… document a bit more

ckan/lib/helpers.py

@@ -37,6 +37,7 @@
 import ckan.lib.maintain as maintain
 import ckan.lib.datapreview as datapreview
 import ckan.logic as logic
+import ckan.lib.uploader as uploader
 
 from ckan.common import (
     _, ungettext, g, c, request, session, json, OrderedDict
@@ -1664,7 +1665,7 @@ def new_activities():
     return action({}, {})
 
 def uploads_enabled():
-    if config.get('ckan.storage_path'):
+    if uploader.get_storage_path():
         return True
     return False
 

ckan/lib/uploader.py

@@ -3,11 +3,51 @@
 import pylons
 import datetime
 import ckan.lib.munge as munge
+import logging
+import ckan.logic as logic
+
+log = logging.getLogger(__name__)
+
+_storage_path = None
+
+def get_storage_path():
+    '''Function to cache storage path'''
+    global _storage_path
+
+    #None means it has not been set. False means not in config.
+    if _storage_path is None:
+        storage_path = pylons.config.get('ckan.storage_path')
+        ofs_impl = pylons.config.get('ofs.impl')
+        ofs_storage_dir = pylons.config.get('ofs.storage_dir')
+        if storage_path:
+            _storage_path = storage_path
+        elif ofs_impl == 'pairtree' and ofs_storage_dir:
+            log.warn('''Please use config option ckan.storage_path instaed of
+                     ofs.storage_path''')
+            _storage_path = ofs_storage_dir
+            return _storage_path
+        elif ofs_impl:
+            log.critical('''We only support local file storage form version 2.2 of ckan
+                          please specify ckan.storage_path in your config for your uploads''')
+            _storage_path = False
+        else:
+            log.critical('''Please specify a ckan.storage_path in your config for your uploads''')
+            _storage_path = False
+
+    return _storage_path
+
 
 
 class Upload(object):
     def __init__(self, object_type, old_filename=None):
-        path = pylons.config.get('ckan.storage_path', '/tmp')
+        ''' Setup upload by creating  a subdirectory of the storage directory of name
+        object_type. old_filename is the name of the file in the url field last time'''
+        self.storage_path = None
+        self.filename = None
+        self.filepath = None
+        path = get_storage_path()
+        if not path:
+            return
         self.storage_path = os.path.join(path, 'storage',
                                          'uploads', object_type)
         try:
@@ -18,12 +58,20 @@ def __init__(self, object_type, old_filename=None):
         self.old_filename = old_filename
         if old_filename:
             self.old_filepath = os.path.join(self.storage_path, old_filename)
-        self.filename = None
-        self.filepath = None
 
     def update_data_dict(self, data_dict, url_field, file_field, clear_field):
+        ''' Manipulate data from the data_dict.  url_field is the name of the field
+        where the upload is going to be. file_field is name of the key where the
+        FieldStorage is kept (i.e the field where the file data actually is). clear_field
+        is the name of a boolean field which requests the upload to be deleted.  This needs
+        to be called before it reaches any validators'''
+
+        if not self.storage_path:
+            return
+
         self.url = data_dict.get(url_field, '')
         self.clear = data_dict.pop(clear_field, None)
+        self.file_field = file_field
         self.upload_field_storage = data_dict.pop(file_field, None)
 
         if isinstance(self.upload_field_storage, cgi.FieldStorage):
@@ -41,16 +89,27 @@ def update_data_dict(self, data_dict, url_field, file_field, clear_field):
             if self.clear and self.url == self.old_filename:
                 data_dict[url_field] = ''
 
-    def upload(self):
+    def upload(self, max_size=2):
+        ''' Actually upload the file.  This should happen just before a commit but after
+        the data has been validated and flushed to the db.  This is so we do not store anything
+        unless the request is actually good. Max_size is size in MB maximum of the file'''
+
         if self.filename:
             output_file = open(self.tmp_filepath, 'wb')
             self.upload_file.seek(0)
+            current_size = 0
             while True:
-                # mb chuncks
+                current_size = current_size + 1
+                # MB chuncks
                 data = self.upload_file.read(2 ** 20)
                 if not data:
                     break
                 output_file.write(data)
+                if current_size > max_size:
+                    os.remove(self.tmp_filepath)
+                    raise logic.ValidationError(
+                        {self.file_field: ['File upload too large']}
+                    )
             output_file.close()
             os.rename(self.tmp_filepath, self.filepath)
             self.clear = True