Merge pull request #1262 from okfn/1262-org-group-image-upload

Upload for group and organization images.

ckan/config/middleware.py

@@ -4,6 +4,7 @@
 import logging
 import json
 import hashlib
+import os
 
 import sqlalchemy as sa
 from beaker.middleware import CacheMiddleware, SessionMiddleware
@@ -22,6 +23,7 @@
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IMiddleware
 from ckan.lib.i18n import get_locales_from_config
+import ckan.lib.uploader as uploader
 
 from ckan.config.environment import load_environment
 import ckan.lib.app_globals as app_globals
@@ -147,6 +149,20 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
                 cache_max_age=static_max_age)
         static_parsers = [static_app, app]
 
+        storage_directory = uploader.get_storage_path()
+        if storage_directory:
+            path = os.path.join(storage_directory, 'storage')
+            try:
+                os.makedirs(path)
+            except OSError, e:
+                ## errno 17 is file already exists
+                if e.errno != 17:
+                    raise
+
+            storage_app = StaticURLParser(path,
+                cache_max_age=static_max_age)
+            static_parsers.insert(0, storage_app)
+
         # Configurable extra static file paths
         extra_static_parsers = []
         for public_path in config.get('extra_public_paths', '').split(','):

ckan/controllers/group.py

@@ -1,6 +1,8 @@
 import re
+import os
 import logging
 import genshi
+import cgi
 import datetime
 from urllib import urlencode
 
@@ -425,6 +427,9 @@ def new(self, data=None, errors=None, error_summary=None):
             return self._save_new(context, group_type)
 
         data = data or {}
+        if not data.get('image_url', '').startswith('http'):
+            data.pop('image_url', None)
+
         errors = errors or {}
         error_summary = error_summary or {}
         vars = {'data': data, 'errors': errors,
@@ -524,7 +529,6 @@ def _save_edit(self, id, context):
             data_dict['id'] = id
             context['allow_partial_update'] = True
             group = self._action('group_update')(context, data_dict)
-
             if id != group['name']:
                 self._force_reindex(group)
 

ckan/lib/dictization/model_dictize.py

@@ -10,6 +10,7 @@
 import ckan.lib.dictization as d
 import ckan.new_authz as new_authz
 import ckan.lib.search as search
+import ckan.lib.munge as munge
 
 ## package save
 
@@ -41,6 +42,16 @@ def group_list_dictize(obj_list, context,
 
         group_dict['display_name'] = obj.display_name
 
+        image_url = group_dict.get('image_url')
+        group_dict['image_display_url'] = image_url
+        if image_url and not image_url.startswith('http'):
+            #munge here should not have an effect only doing it incase
+            #of potential vulnerability of dodgy api input
+            image_url = munge.munge_filename(image_url)
+            group_dict['image_display_url'] = h.url_for_static(
+                'uploads/group/%s' % group_dict.get('image_url')
+            )
+
         if obj.is_organization:
             group_dict['packages'] = query.facets['owner_org'].get(obj.id, 0)
         else:
@@ -357,6 +368,16 @@ def group_dictize(group, context):
         for item in plugins.PluginImplementations(plugin):
             result_dict = item.before_view(result_dict)
 
+    image_url = result_dict.get('image_url')
+    result_dict['image_display_url'] = image_url
+    if image_url and not image_url.startswith('http'):
+        #munge here should not have an effect only doing it incase
+        #of potential vulnerability of dodgy api input
+        image_url = munge.munge_filename(image_url)
+        result_dict['image_display_url'] = h.url_for_static(
+            'uploads/group/%s' % result_dict.get('image_url'),
+            qualified = True
+        )
     return result_dict
 
 def tag_list_dictize(tag_list, context):

ckan/lib/helpers.py

@@ -37,6 +37,7 @@
 import ckan.lib.maintain as maintain
 import ckan.lib.datapreview as datapreview
 import ckan.logic as logic
+import ckan.lib.uploader as uploader
 
 from ckan.common import (
     _, ungettext, g, c, request, session, json, OrderedDict
@@ -1664,6 +1665,10 @@ def new_activities():
     action = logic.get_action('dashboard_new_activities_count')
     return action({}, {})
 
+def uploads_enabled():
+    if uploader.get_storage_path():
+        return True
+    return False
 
 def get_featured_organizations(count=1):
     '''Returns a list of favourite organization in the form
@@ -1836,6 +1841,7 @@ def get_site_statistics():
     'radio',
     'submit',
     'asbool',
+    'uploads_enabled',
     'get_featured_organizations',
     'get_featured_groups',
     'get_site_statistics',

ckan/lib/munge.py

@@ -105,6 +105,13 @@ def munge_tag(tag):
     tag = _munge_to_length(tag, model.MIN_TAG_LENGTH, model.MAX_TAG_LENGTH)
     return tag
 
+def munge_filename(filename):
+    filename = substitute_ascii_equivalents(filename)
+    filename = filename.lower().strip()
+    filename = re.sub(r'[^a-zA-Z0-9. ]', '', filename).replace(' ', '-')
+    filename = _munge_to_length(filename, 3, 100)
+    return filename
+
 def _munge_to_length(string, min_length, max_length):
     '''Pad/truncates a string'''
     if len(string) < min_length:

ckan/lib/uploader.py

@@ -0,0 +1,131 @@
+import os
+import cgi
+import pylons
+import datetime
+import ckan.lib.munge as munge
+import logging
+import ckan.logic as logic
+
+log = logging.getLogger(__name__)
+
+_storage_path = None
+
+
+def get_storage_path():
+    '''Function to cache storage path'''
+    global _storage_path
+
+    #None means it has not been set. False means not in config.
+    if _storage_path is None:
+        storage_path = pylons.config.get('ckan.storage_path')
+        ofs_impl = pylons.config.get('ofs.impl')
+        ofs_storage_dir = pylons.config.get('ofs.storage_dir')
+        if storage_path:
+            _storage_path = storage_path
+        elif ofs_impl == 'pairtree' and ofs_storage_dir:
+            log.warn('''Please use config option ckan.storage_path instaed of
+                     ofs.storage_path''')
+            _storage_path = ofs_storage_dir
+            return _storage_path
+        elif ofs_impl:
+            log.critical('''We only support local file storage form version 2.2
+                         of ckan please specify ckan.storage_path in your
+                         config for your uploads''')
+            _storage_path = False
+        else:
+            log.critical('''Please specify a ckan.storage_path in your config
+                         for your uploads''')
+            _storage_path = False
+
+    return _storage_path
+
+
+class Upload(object):
+    def __init__(self, object_type, old_filename=None):
+        ''' Setup upload by creating  a subdirectory of the storage directory
+        of name object_type. old_filename is the name of the file in the url
+        field last time'''
+
+        self.storage_path = None
+        self.filename = None
+        self.filepath = None
+        path = get_storage_path()
+        if not path:
+            return
+        self.storage_path = os.path.join(path, 'storage',
+                                         'uploads', object_type)
+        try:
+            os.makedirs(self.storage_path)
+        except OSError, e:
+            ## errno 17 is file already exists
+            if e.errno != 17:
+                raise
+        self.object_type = object_type
+        self.old_filename = old_filename
+        if old_filename:
+            self.old_filepath = os.path.join(self.storage_path, old_filename)
+
+    def update_data_dict(self, data_dict, url_field, file_field, clear_field):
+        ''' Manipulate data from the data_dict.  url_field is the name of the
+        field where the upload is going to be. file_field is name of the key
+        where the FieldStorage is kept (i.e the field where the file data
+        actually is). clear_field is the name of a boolean field which
+        requests the upload to be deleted.  This needs to be called before
+        it reaches any validators'''
+
+        self.url = data_dict.get(url_field, '')
+        self.clear = data_dict.pop(clear_field, None)
+        self.file_field = file_field
+        self.upload_field_storage = data_dict.pop(file_field, None)
+
+        if not self.storage_path:
+            return
+
+        if isinstance(self.upload_field_storage, cgi.FieldStorage):
+            self.filename = self.upload_field_storage.filename
+            self.filename = str(datetime.datetime.utcnow()) + self.filename
+            self.filename = munge.munge_filename(self.filename)
+            self.filepath = os.path.join(self.storage_path, self.filename)
+            data_dict[url_field] = self.filename
+            self.upload_file = self.upload_field_storage.file
+            self.tmp_filepath = self.filepath + '~'
+        ### keep the file if there has been no change
+        elif self.old_filename and not self.old_filename.startswith('http'):
+            if not self.clear:
+                data_dict[url_field] = self.old_filename
+            if self.clear and self.url == self.old_filename:
+                data_dict[url_field] = ''
+
+    def upload(self, max_size=2):
+        ''' Actually upload the file.
+        This should happen just before a commit but after the data has
+        been validated and flushed to the db. This is so we do not store
+        anything unless the request is actually good.
+        max_size is size in MB maximum of the file'''
+
+        if self.filename:
+            output_file = open(self.tmp_filepath, 'wb')
+            self.upload_file.seek(0)
+            current_size = 0
+            while True:
+                current_size = current_size + 1
+                # MB chuncks
+                data = self.upload_file.read(2 ** 20)
+                if not data:
+                    break
+                output_file.write(data)
+                if current_size > max_size:
+                    os.remove(self.tmp_filepath)
+                    raise logic.ValidationError(
+                        {self.file_field: ['File upload too large']}
+                    )
+            output_file.close()
+            os.rename(self.tmp_filepath, self.filepath)
+            self.clear = True
+
+        if (self.clear and self.old_filename
+                and not self.old_filename.startswith('http')):
+            try:
+                os.remove(self.old_filepath)
+            except OSError, e:
+                pass

ckan/logic/action/create.py

@@ -17,6 +17,7 @@
 import ckan.lib.dictization.model_dictize as model_dictize
 import ckan.lib.dictization.model_save as model_save
 import ckan.lib.navl.dictization_functions
+import ckan.lib.uploader as uploader
 import ckan.lib.navl.validators as validators
 import ckan.lib.mailer as mailer
 
@@ -450,6 +451,7 @@ def member_create(context, data_dict=None):
     if not obj:
         raise NotFound('%s was not found.' % obj_type.title())
 
+
     # User must be able to update the group to add a member to it
     _check_access('group_update', context, data_dict)
 
@@ -479,7 +481,9 @@ def _group_or_org_create(context, data_dict, is_org=False):
     parent = context.get('parent', None)
     data_dict['is_organization'] = is_org
 
-
+    upload = uploader.Upload('group')
+    upload.update_data_dict(data_dict, 'image_url',
+                           'image_upload', 'clear_upload')
     # get the schema
     group_plugin = lib_plugins.lookup_group_plugin(
             group_type=data_dict.get('type'))
@@ -565,6 +569,7 @@ def _group_or_org_create(context, data_dict, is_org=False):
     logic.get_action('activity_create')(activity_create_context,
             activity_dict)
 
+    upload.upload()
     if not context.get('defer_commit'):
         model.repo.commit()
     context["group"] = group

ckan/logic/action/update.py

@@ -19,6 +19,7 @@
 import ckan.lib.plugins as lib_plugins
 import ckan.lib.email_notifications as email_notifications
 import ckan.lib.search as search
+import ckan.lib.uploader as uploader
 
 from ckan.common import _, request
 
@@ -424,6 +425,10 @@ def _group_or_org_update(context, data_dict, is_org=False):
     except AttributeError:
         schema = group_plugin.form_to_db_schema()
 
+    upload = uploader.Upload('group', group.image_url)
+    upload.update_data_dict(data_dict, 'image_url',
+                           'image_upload', 'clear_upload')
+
     if is_org:
         _check_access('organization_update', context, data_dict)
     else:
@@ -528,9 +533,11 @@ def _group_or_org_update(context, data_dict, is_org=False):
         # TODO: Also create an activity detail recording what exactly changed
         # in the group.
 
+    upload.upload()
     if not context.get('defer_commit'):
         model.repo.commit()
 
+
     return model_dictize.group_dictize(group, context)
 
 def group_update(context, data_dict):

ckan/logic/schema.py

@@ -263,6 +263,7 @@ def default_group_schema():
         'title': [ignore_missing, unicode],
         'description': [ignore_missing, unicode],
         'image_url': [ignore_missing, unicode],
+        'image_display_url': [ignore_missing, unicode],
         'type': [ignore_missing, unicode],
         'state': [ignore_not_group_admin, ignore_missing],
         'created': [ignore],