From 35acb45361c3e2eb750cf7f54d64023e5e9f728b Mon Sep 17 00:00:00 2001 From: kindly Date: Wed, 13 Nov 2013 13:58:38 +0000 Subject: [PATCH] [#1273] add size limits properly and add docs --- ckan/lib/uploader.py | 24 +++++++-- ckan/logic/action/create.py | 5 +- ckan/logic/action/update.py | 4 +- doc/configuration.rst | 103 +++++++++++------------------------- 4 files changed, 56 insertions(+), 80 deletions(-) diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 2a348d7150d..27c998dc24c 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -6,9 +6,13 @@ import logging import ckan.logic as logic + +config = pylons.config log = logging.getLogger(__name__) _storage_path = None +_max_resource_size = None +_max_image_size = None def get_storage_path(): @@ -17,9 +21,9 @@ def get_storage_path(): #None means it has not been set. False means not in config. if _storage_path is None: - storage_path = pylons.config.get('ckan.storage_path') - ofs_impl = pylons.config.get('ofs.impl') - ofs_storage_dir = pylons.config.get('ofs.storage_dir') + storage_path = config.get('ckan.storage_path') + ofs_impl = config.get('ofs.impl') + ofs_storage_dir = config.get('ofs.storage_dir') if storage_path: _storage_path = storage_path elif ofs_impl == 'pairtree' and ofs_storage_dir: @@ -39,6 +43,18 @@ def get_storage_path(): return _storage_path +def get_max_image_size(): + global _max_image_size + if _max_image_size is None: + _max_image_size = int(config.get('ckan.max_image_size', 2)) + return _max_image_size + +def get_max_resource_size(): + global _max_resource_size + if _max_resource_size is None: + _max_resource_size = int(config.get('ckan.max_resource_size', 10)) + return _max_resource_size + class Upload(object): def __init__(self, object_type, old_filename=None): @@ -191,7 +207,7 @@ def upload(self, id, max_size=10): break output_file.write(data) if current_size > max_size: - os.remove(self.tmp_filepath) + os.remove(tmp_filepath) raise logic.ValidationError( {'upload': ['File upload too large']} ) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index c419cafc0d5..652a87d2f7d 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -275,7 +275,8 @@ def resource_create(context, data_dict): ## Get out resource_id resource from model as it will not appear in ## package_show until after commit - upload.upload(context['package'].resources[-1].id) + upload.upload(context['package'].resources[-1].id, + uploader.get_max_resource_size()) model.repo.commit() ## Run package show again to get out actual last_resource @@ -587,7 +588,7 @@ def _group_or_org_create(context, data_dict, is_org=False): logic.get_action('activity_create')(activity_create_context, activity_dict) - upload.upload() + upload.upload(uploader.get_max_image_size()) if not context.get('defer_commit'): model.repo.commit() context["group"] = group diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 505003e0920..e1b59b5b072 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -233,7 +233,7 @@ def resource_update(context, data_dict): raise ValidationError(errors) resource = pkg_dict['resources'][n] - upload.upload(resource['id']) + upload.upload(resource['id'], uploader.get_max_resource_size()) model.repo.commit() return resource @@ -541,7 +541,7 @@ def _group_or_org_update(context, data_dict, is_org=False): # TODO: Also create an activity detail recording what exactly changed # in the group. - upload.upload() + upload.upload(uploader.get_max_image_size()) if not context.get('defer_commit'): model.repo.commit() diff --git a/doc/configuration.rst b/doc/configuration.rst index ec113ea9d4a..01b18660f92 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -971,20 +971,39 @@ For more information on theming, see :doc:`theming`. Storage Settings ---------------- -.. _ckan.storage.bucket: +.. _ckan.storage_path: -ckan.storage.bucket -^^^^^^^^^^^^^^^^^^^ +ckan.storage_path +^^^^^^^^^^^^^^^^^ Example:: - - ckan.storage.bucket = ckan + ckan.storage_path = /var/lib/ckan Default value: ``None`` -This changes the bucket name for the uploaded files. +This defines the location of where CKAN will store all uploaded data. + +ckan.max_resource_size +^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + ckan.max_resource_size = 100 + +Default value: ``10`` + +The maximum in megabytes a resources upload can be. + +ckan.max_image_size +^^^^^^^^^^^^^^^^^^^^ + +Example:: + ckan.max_image_size = 10 + +Default value: ``2`` -.. _ckan.storage.max_content_length: +The maximum in megabytes an image upload can be. + +.. _ckan.storage.bucket: ckan.storage.max_content_length ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -996,6 +1015,7 @@ Example:: Default value: ``50000000`` This defines the maximum content size, in bytes, for uploads. +Depricated, please use ckan.max_resource_size_instead .. _ofs.impl: @@ -1010,6 +1030,9 @@ Default value: ``None`` Defines the storage backend used by CKAN: ``pairtree`` for local storage, ``s3`` for Amazon S3 Cloud Storage or ``google`` for Google Cloud Storage. Note that each of these must be accompanied by the relevant settings for each backend described below. +Depricated, only aviliable option is now pairtree. + + .. _ofs.storage_dir: ofs.storage_dir @@ -1023,72 +1046,8 @@ Default value: ``None`` Only used with the local storage backend. Use this to specify where uploaded files should be stored, and also to turn on the handling of file storage. The folder should exist, and will automatically be turned into a valid pairtree repository if it is not already. -.. _ckan.storage.key_prefix: - -ckan.storage.key_prefix -^^^^^^^^^^^^^^^^^^^^^^^ - -Example:: - - ckan.storage.key_prefix = ckan-file/ - -Default value: ``file/`` - -Only used with the local storage backend. This changes the prefix for the uploaded files. - -.. _ofs.aws_access_key_id: - -ofs.aws_access_key_id -^^^^^^^^^^^^^^^^^^^^^ - -Example:: - - ofs.aws_access_key_id = 022QF06E7MXBSH9DHM02 - -Default value: ``None`` - -Only used with the Amazon S3 storage backend. Configure with your AWS Access Key ID. - -.. _ofs.aws_secret_access_key: - -ofs.aws_secret_access_key -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example:: - - ofs.aws_secret_access_key = kWcrlUX5JEDGM/LtmEENI/aVmYvHNif5zB+d9+ct - -Default value: ``None`` - -Only used with the Amazon S3 storage backend. Configure with your AWS Secret Access Key. - -.. _ofs.gs_access_key_id: - -ofs.gs_access_key_id -^^^^^^^^^^^^^^^^^^^^^ - -Example:: - - ofs.gs_access_key_id = GOOGTS7C7FUP3AIRVJTE - -Default value: ``None`` - -Only used with the Google storage backend. Configure with your Google Storage -Access Key ID. - -.. _ofs.gs_secret_access_key: - -ofs.gs_secret_access_key -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example:: - - ofs.gs_secret_access_key = bGoa+V7g/yqDXvKRqq+JTFn4uQZbPiQJo4pf9RzJ - -Default value: ``None`` +Depricated, please use ckan.storage_path -Only used with the Google storage backend. Configure with your Google Storage -Secret Access Key. DataPusher Settings