Merge branch 'master' of github.com:okfn/ckan into feature-1698-tag-t…

…axonomies

Conflicts:
	ckan/lib/dictization/model_dictize.py
	ckan/lib/dictization/model_save.py
	ckan/logic/action/create.py
	ckan/logic/action/get.py
	ckan/logic/auth/create.py
	ckan/logic/schema.py
	ckan/logic/validators.py
	ckan/model/__init__.py
	ckan/model/package.py
	ckan/model/tag.py
	ckan/tests/lib/test_dictization.py

ckan/config/routing.py

@@ -8,14 +8,17 @@
 from pylons import config
 from routes import Mapper
 from ckan.plugins import PluginImplementations, IRoutes
-from ckan.controllers.package import register_pluggable_behaviour as register_package_behaviour
-from ckan.controllers.group   import register_pluggable_behaviour as register_group_behaviour
 
 
 routing_plugins = PluginImplementations(IRoutes)
 
 def make_map():
     """Create, configure and return the routes Mapper"""
+    # import controllers here rather than at root level because 
+    # pylons config is initialised by this point.
+    from ckan.controllers.package import register_pluggable_behaviour as register_package_behaviour
+    from ckan.controllers.group   import register_pluggable_behaviour as register_group_behaviour
+
     map = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])
     map.minimization = False
@@ -46,7 +49,9 @@ def make_map():
             'authorizationgroup',
             'revision',
             'licenses',
-            'rating'
+            'rating',
+            'user',
+            'activity'
             ]
     register_list_str = '|'.join(register_list)
 
@@ -273,6 +278,7 @@ def make_map():
     map.connect('/tag/{id}', controller='tag', action='read')
     # users
     map.redirect("/users/{url:.*}", "/user/{url}")
+    map.redirect("/user/", "/user")
     map.connect('/user/edit', controller='user', action='edit')
     # Note: openid users have slashes in their ids, so need the wildcard
     # in the route.

ckan/controllers/api.py

@@ -14,7 +14,7 @@
 from ckan.lib.navl.dictization_functions import DataError
 from ckan.lib.munge import munge_name, munge_title_to_name, munge_tag
 from ckan.logic import get_action, check_access
-from ckan.logic import NotFound, NotAuthorized, ValidationError
+from ckan.logic import NotFound, NotAuthorized, ValidationError, ParameterError
 from ckan.lib.jsonp import jsonpify
 from ckan.forms.common import package_exists, group_exists
 
@@ -170,9 +170,11 @@ def action(self, logic_function):
                                     'message': _('Access denied')}
             return_dict['success'] = False
             return self._finish(403, return_dict, content_type='json')
-        except NotFound:
+        except NotFound, e:
             return_dict['error'] = {'__type': 'Not Found Error',
                                     'message': _('Not found')}
+            if e.extra_msg:
+                return_dict['error']['message'] += ': %s' % e.extra_msg
             return_dict['success'] = False
             return self._finish(404, return_dict, content_type='json')
         except ValidationError, e:
@@ -182,6 +184,13 @@ def action(self, logic_function):
             return_dict['success'] = False
             log.error('Validation error: %r' % str(e.error_dict))
             return self._finish(409, return_dict, content_type='json')
+        except ParameterError, e:
+            return_dict['error'] = {'__type': 'Parameter Error',
+                                    'message': '%s: %s' % \
+                                    (_('Parameter Error'), e.extra_msg)}
+            return_dict['success'] = False
+            log.error('Parameter error: %r' % e.extra_msg)
+            return self._finish(409, return_dict, content_type='json')            
         except SearchQueryError, e:
             return_dict['error'] = {'__type': 'Search Query Error',
                                     'message': 'Search Query is invalid: %r' % e.args }
@@ -209,6 +218,11 @@ def list(self, ver=None, register=None, subregister=None, id=None):
             ('package', 'relationships'): get_action('package_relationships_list'),
             ('dataset', 'revisions'): get_action('package_revision_list'),
             ('package', 'revisions'): get_action('package_revision_list'),
+            ('package', 'activity'): get_action('package_activity_list'),
+            ('dataset', 'activity'): get_action('package_activity_list'),
+            ('group', 'activity'): get_action('group_activity_list'),
+            ('user', 'activity'): get_action('user_activity_list'),
+            ('activity', 'details'): get_action('activity_detail_list')
         }
 
         action = action_map.get((register, subregister)) 

ckan/controllers/authorization_group.py

@@ -157,23 +157,11 @@ def authz(self, id):
         if not c.authz_editable:
             abort(401, gettext('User %r not authorized to edit %s authorizations') % (c.user, id))
 
-        current_uors = self._get_userobjectroles(id)
-        self._handle_update_of_authz(current_uors, authorization_group)
-
-        # get the roles again as may have changed
-        user_object_roles = self._get_userobjectroles(id)
-        self._prepare_authz_info_for_render(user_object_roles)
+        roles = self._handle_update_of_authz(authorization_group)
+        self._prepare_authz_info_for_render(roles)
         return render('authorization_group/authz.html')
 
 
-    def _get_userobjectroles(self, authzgroup_name_or_id):
-        authorization_group = model.AuthorizationGroup.by_name(authzgroup_name_or_id)  or\
-               model.Session.query(model.AuthorizationGroup).get(authzgroup_name_or_id)
-        uors = model.Session.query(model.AuthorizationGroupRole).join('authorization_group').\
-               filter_by(id=authorization_group.id).all()
-        return uors
-
-
     def _render_edit_form(self, fs):
         # errors arrive in c.error and fs.errors
         c.fieldset = fs

ckan/controllers/group.py

@@ -1,3 +1,4 @@
+import logging
 import genshi
 import datetime
 from urllib import urlencode
@@ -17,7 +18,9 @@
 from ckan.logic import tuplize_dict, clean_dict, parse_params
 from ckan.lib.dictization.model_dictize import package_dictize
 import ckan.forms
+import ckan.logic.action.get
 
+log = logging.getLogger(__name__)
 
 # Mapping from group-type strings to IDatasetForm instances
 _controller_behaviour_for = dict()
@@ -302,6 +305,12 @@ def pager_url(q=None, page=None):
             c.query_error = True
             c.facets = {}
             c.page = h.Page(collection=[])
+
+        # Add the group's activity stream (already rendered to HTML) to the
+        # template context for the group/read.html template to retrieve later.
+        c.group_activity_stream = \
+                ckan.logic.action.get.group_activity_list_html(context,
+                    {'id': c.group_dict['id']})
 
         return render('group/read.html')
 
@@ -458,21 +467,11 @@ def authz(self, id):
         if not c.authz_editable:
             abort(401, gettext('User %r not authorized to edit %s authorizations') % (c.user, id))
 
-        current_uors = self._get_userobjectroles(id)
-        self._handle_update_of_authz(current_uors, group)
-
-        # get the roles again as may have changed
-        user_object_roles = self._get_userobjectroles(id)
-        self._prepare_authz_info_for_render(user_object_roles)
+        roles = self._handle_update_of_authz(group)
+        self._prepare_authz_info_for_render(roles)
         return render('group/authz.html')
 
 
-    def _get_userobjectroles(self, group_id):
-        group = model.Group.get(group_id)
-        uors = model.Session.query(model.GroupRole).join('group').filter_by(name=group.name).all()
-        return uors
-
-
     def history(self, id):
         if 'diff' in request.params or 'selected1' in request.params:
             try:

ckan/controllers/package.py

@@ -29,8 +29,9 @@
 import ckan.authz
 import ckan.rating
 import ckan.misc
+import ckan.logic.action.get
 
-log = logging.getLogger('ckan.controllers')
+log = logging.getLogger(__name__)
 
 def search_url(params):
     url = h.url_for(controller='package', action='search')
@@ -108,7 +109,6 @@ def _lookup_plugin(package_type):
     If the package type is None or cannot be found in the mapping, then the
     fallback behaviour is used.
     """
-    #from pdb import set_trace; set_trace()    
     if package_type is None:
         return _default_controller_behaviour
     return _controller_behaviour_for.get(package_type,
@@ -325,7 +325,14 @@ def read(self, id):
 
         # used by disqus plugin
         c.current_package_id = c.pkg.id
-
+
+        # Add the package's activity stream (already rendered to HTML) to the
+        # template context for the package/read.html template to retrieve
+        # later.
+        c.package_activity_stream = \
+                ckan.logic.action.get.package_activity_list_html(context,
+                    {'id': c.current_package_id})
+
         if config.get('rdf_packages'):
             accept_header = request.headers.get('Accept', '*/*')
             for content_type, exts in negotiate(autoneg_cfg, accept_header):
@@ -698,21 +705,10 @@ def authz(self, id):
         if not c.authz_editable:
             abort(401, gettext('User %r not authorized to edit %s authorizations') % (c.user, id))
 
-        current_uors = self._get_userobjectroles(id)
-        self._handle_update_of_authz(current_uors, pkg)
-
-        # get the roles again as may have changed
-        user_object_roles = self._get_userobjectroles(id)
-        self._prepare_authz_info_for_render(user_object_roles)
+        roles = self._handle_update_of_authz(pkg)
+        self._prepare_authz_info_for_render(roles)
         return render('package/authz.html')
 
-
-    def _get_userobjectroles(self, pkg_id):
-        pkg = model.Package.get(pkg_id)
-        uors = model.Session.query(model.PackageRole).join('package').filter_by(name=pkg.name).all()
-        return uors
-
-
     def autocomplete(self):
         # DEPRECATED in favour of /api/2/util/dataset/autocomplete
         q = unicode(request.params.get('q', ''))

ckan/controllers/storage.py

@@ -30,10 +30,8 @@
 log = getLogger(__name__)
 
 
-# pairtree_version0_1 file for identifying folders
 BUCKET = config.get('ckan.storage.bucket', 'default')
 key_prefix = config.get('ckan.storage.key_prefix', 'file/')
-storage_dir = config.get('ckan.storage.directory', '')
 
 _eq_re = re.compile(r"^(.*)(=[0-9]*)$")
 def fix_stupid_pylons_encoding(data):
@@ -50,30 +48,16 @@ def fix_stupid_pylons_encoding(data):
 
 
 def get_ofs():
+    """Return a configured instance of the appropriate OFS driver.
     """
-    Return a configured instance of the appropriate OFS driver, in all
-    cases here this will be the local file storage so we fix the implementation
-    to use pairtree.
-    """
-    return get_impl("pairtree")(storage_dir=storage_dir)
-
-
-pairtree_marker_done = False
-def create_pairtree_marker():
-    """
-    Make sure that the file pairtree_version0_1 is present in storage_dir
-    and if not then create it.
-    """
-    global pairtree_marker_done
-    if pairtree_marker_done or not storage_dir:
-        return
-
-    path = os.path.join(storage_dir, 'pairtree_version0_1')
-    if not os.path.exists( path ):
-        open(path, 'w').close() 
-
-    pairtree_marker_done = True
-
+    storage_backend = config['ofs.impl']
+    kw = {}
+    for k,v in config.items():
+        if not k.startswith('ofs.') or k == 'ofs.impl':
+            continue
+        kw[k[4:]] = v
+    ofs = get_impl(storage_backend)(**kw)
+    return ofs
 
 
 def authorize(method, bucket, key, user, ofs):
@@ -99,13 +83,6 @@ def authorize(method, bucket, key, user, ofs):
 class StorageController(BaseController):
     '''Upload to storage backend.
     '''
-    def __before__(self, action, **params):
-        super(StorageController, self).__before__(action, **params)
-        if not storage_dir:
-            abort(404)
-        else:
-            create_pairtree_marker()            
-
     @property
     def ofs(self):
         return get_ofs()
@@ -188,14 +165,6 @@ def file(self, label):
 
 
 class StorageAPIController(BaseController):
-
-    def __before__(self, action, **params):
-        super(StorageAPIController, self).__before__(action, **params)
-        if not storage_dir:
-            abort(404)
-        else:
-            create_pairtree_marker()
-
     @property
     def ofs(self):
         return get_ofs()
@@ -260,10 +229,16 @@ def set_metadata(self, label):
     @jsonpify
     def get_metadata(self, label):
         bucket = BUCKET
-        url = h.url_for('storage_file',
-                label=label,
-                qualified=True
-                )
+        storage_backend = config['ofs.impl']
+        if storage_backend in ['google', 's3']:
+            if not label.startswith("/"):
+                label = "/" + label
+            url = "https://%s/%s%s" % (self.ofs.conn.server_name(), bucket, label)
+        else:
+            url = h.url_for('storage_file',
+                    label=label,
+                    qualified=True
+                    )
         if not self.ofs.exists(bucket, label):
             abort(404)
         metadata = self.ofs.get_metadata(bucket, label)
@@ -331,7 +306,7 @@ def _get_remote_form_data(self, label):
         acl = 'public-read'
         fields = [ {
                 'name': self.ofs.conn.provider.metadata_prefix + 'uploaded-by',
-                'value': c.userobj.name
+                'value': c.userobj.id
                 }]
         conditions = [ '{"%s": "%s"}' % (x['name'], x['value']) for x in
                 fields ]
@@ -353,22 +328,30 @@ def _get_remote_form_data(self, label):
             )
         # HACK: fix up some broken stuff from boto
         # e.g. should not have content-length-range in list of fields!
+        storage_backend = config['ofs.impl']
         for idx,field in enumerate(data['fields']):
+            if storage_backend == 'google':
+                if field['name'] == 'AWSAccessKeyId':
+                    field['name'] = 'GoogleAccessId'
             if field['name'] == 'content-length-range':
                 del data['fields'][idx]
         return data
 
     def _get_form_data(self, label):
-        data = {
-            'action': h.url_for('storage_upload_handle', qualified=True),
-            'fields': [
-                {
-                    'name': 'key',
-                    'value': label
-                }
-            ]
-        }
-        return data
+        storage_backend = config['ofs.impl']
+        if storage_backend in ['google', 's3']:
+            return self._get_remote_form_data(label)
+        else:
+            data = {
+                'action': h.url_for('storage_upload_handle', qualified=True),
+                'fields': [
+                    {
+                        'name': 'key',
+                        'value': label
+                    }
+                ]
+            }
+            return data
 
     @jsonpify
     def auth_form(self, label):