[#1273] add size limits properly and add docs

ckan/lib/uploader.py

@@ -6,9 +6,13 @@
 import logging
 import ckan.logic as logic
 
+
+config = pylons.config
 log = logging.getLogger(__name__)
 
 _storage_path = None
+_max_resource_size = None
+_max_image_size = None
 
 
 def get_storage_path():
@@ -17,9 +21,9 @@ def get_storage_path():
 
     #None means it has not been set. False means not in config.
     if _storage_path is None:
-        storage_path = pylons.config.get('ckan.storage_path')
-        ofs_impl = pylons.config.get('ofs.impl')
-        ofs_storage_dir = pylons.config.get('ofs.storage_dir')
+        storage_path = config.get('ckan.storage_path')
+        ofs_impl = config.get('ofs.impl')
+        ofs_storage_dir = config.get('ofs.storage_dir')
         if storage_path:
             _storage_path = storage_path
         elif ofs_impl == 'pairtree' and ofs_storage_dir:
@@ -39,6 +43,18 @@ def get_storage_path():
 
     return _storage_path
 
+def get_max_image_size():
+    global _max_image_size
+    if _max_image_size is None:
+        _max_image_size = int(config.get('ckan.max_image_size', 2))
+    return _max_image_size
+
+def get_max_resource_size():
+    global _max_resource_size
+    if _max_resource_size is None:
+        _max_resource_size = int(config.get('ckan.max_resource_size', 10))
+    return _max_resource_size
+
 
 class Upload(object):
     def __init__(self, object_type, old_filename=None):
@@ -191,7 +207,7 @@ def upload(self, id, max_size=10):
                     break
                 output_file.write(data)
                 if current_size > max_size:
-                    os.remove(self.tmp_filepath)
+                    os.remove(tmp_filepath)
                     raise logic.ValidationError(
                         {'upload': ['File upload too large']}
                     )

ckan/logic/action/create.py

@@ -275,7 +275,8 @@ def resource_create(context, data_dict):
 
     ## Get out resource_id resource from model as it will not appear in
     ## package_show until after commit
-    upload.upload(context['package'].resources[-1].id)
+    upload.upload(context['package'].resources[-1].id,
+                  uploader.get_max_resource_size())
     model.repo.commit()
 
     ##  Run package show again to get out actual last_resource
@@ -587,7 +588,7 @@ def _group_or_org_create(context, data_dict, is_org=False):
     logic.get_action('activity_create')(activity_create_context,
             activity_dict)
 
-    upload.upload()
+    upload.upload(uploader.get_max_image_size())
     if not context.get('defer_commit'):
         model.repo.commit()
     context["group"] = group

ckan/logic/action/update.py

@@ -233,7 +233,7 @@ def resource_update(context, data_dict):
         raise ValidationError(errors)
 
     resource = pkg_dict['resources'][n]
-    upload.upload(resource['id'])
+    upload.upload(resource['id'], uploader.get_max_resource_size())
     model.repo.commit()
     return resource
 
@@ -541,7 +541,7 @@ def _group_or_org_update(context, data_dict, is_org=False):
         # TODO: Also create an activity detail recording what exactly changed
         # in the group.
 
-    upload.upload()
+    upload.upload(uploader.get_max_image_size())
     if not context.get('defer_commit'):
         model.repo.commit()
 

doc/configuration.rst

@@ -971,20 +971,39 @@ For more information on theming, see :doc:`theming`.
 Storage Settings
 ----------------
 
-.. _ckan.storage.bucket:
+.. _ckan.storage_path:
 
-ckan.storage.bucket
-^^^^^^^^^^^^^^^^^^^
+ckan.storage_path
+^^^^^^^^^^^^^^^^^
 
 Example::
-
-  ckan.storage.bucket = ckan
+    ckan.storage_path = /var/lib/ckan
 
 Default value:  ``None``
 
-This changes the bucket name for the uploaded files.
+This defines the location of where CKAN will store all uploaded data.
+
+ckan.max_resource_size
+^^^^^^^^^^^^^^^^^^^^^^
+
+Example::
+    ckan.max_resource_size = 100
+
+Default value: ``10``
+
+The maximum in megabytes a resources upload can be.
+
+ckan.max_image_size
+^^^^^^^^^^^^^^^^^^^^
+
+Example::
+    ckan.max_image_size = 10
+
+Default value: ``2``
 
-.. _ckan.storage.max_content_length:
+The maximum in megabytes an image upload can be.
+
+.. _ckan.storage.bucket:
 
 ckan.storage.max_content_length
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -996,6 +1015,7 @@ Example::
 Default value: ``50000000``
 
 This defines the maximum content size, in bytes, for uploads.
+Depricated, please use ckan.max_resource_size_instead
 
 .. _ofs.impl:
 
@@ -1010,6 +1030,9 @@ Default value:  ``None``
 
 Defines the storage backend used by CKAN: ``pairtree`` for local storage, ``s3`` for Amazon S3 Cloud Storage or ``google`` for Google Cloud Storage. Note that each of these must be accompanied by the relevant settings for each backend described below.
 
+Depricated, only aviliable option is now pairtree.
+
+
 .. _ofs.storage_dir:
 
 ofs.storage_dir
@@ -1023,72 +1046,8 @@ Default value:  ``None``
 
 Only used with the local storage backend. Use this to specify where uploaded files should be stored, and also to turn on the handling of file storage. The folder should exist, and will automatically be turned into a valid pairtree repository if it is not already.
 
-.. _ckan.storage.key_prefix:
-
-ckan.storage.key_prefix
-^^^^^^^^^^^^^^^^^^^^^^^
-
-Example::
-
-  ckan.storage.key_prefix = ckan-file/
-
-Default value: ``file/``
-
-Only used with the local storage backend. This changes the prefix for the uploaded files.
-
-.. _ofs.aws_access_key_id:
-
-ofs.aws_access_key_id
-^^^^^^^^^^^^^^^^^^^^^
-
-Example::
-
-  ofs.aws_access_key_id = 022QF06E7MXBSH9DHM02
-
-Default value:  ``None``
-
-Only used with the Amazon S3 storage backend. Configure with your AWS Access Key ID.
-
-.. _ofs.aws_secret_access_key:
-
-ofs.aws_secret_access_key
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Example::
-
-  ofs.aws_secret_access_key = kWcrlUX5JEDGM/LtmEENI/aVmYvHNif5zB+d9+ct
-
-Default value:  ``None``
-
-Only used with the Amazon S3 storage backend. Configure with your AWS Secret Access Key.
-
-.. _ofs.gs_access_key_id:
-
-ofs.gs_access_key_id
-^^^^^^^^^^^^^^^^^^^^^
-
-Example::
-
-  ofs.gs_access_key_id = GOOGTS7C7FUP3AIRVJTE
-
-Default value:  ``None``
-
-Only used with the Google storage backend. Configure with your Google Storage
-Access Key ID.
-
-.. _ofs.gs_secret_access_key:
-
-ofs.gs_secret_access_key
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Example::
-
-  ofs.gs_secret_access_key = bGoa+V7g/yqDXvKRqq+JTFn4uQZbPiQJo4pf9RzJ
-
-Default value:  ``None``
+Depricated, please use ckan.storage_path
 
-Only used with the Google storage backend. Configure with your Google Storage
-Secret Access Key.
 
 
 DataPusher Settings