Merge branch 'master' into 1255-better-search-results

ckan/config/deployment.ini_tmpl

@@ -67,6 +67,7 @@ ckan.auth.user_delete_groups = true
 ckan.auth.user_delete_organizations = true
 ckan.auth.create_user_via_api = false
 ckan.auth.create_user_via_web = true
+ckan.auth.roles_that_cascade_to_sub_groups = admin
 
 
 ## Search Settings

ckan/config/environment.py

@@ -367,8 +367,10 @@ def template_loaded(template):
     # Here we create the site user if they are not already in the database
     try:
         logic.get_action('get_site_user')({'ignore_auth': True}, None)
-    except sqlalchemy.exc.ProgrammingError:
-        # The database is not initialised.  This is a bit dirty.
+    except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
+        # (ProgrammingError for Postgres, OperationalError for SQLite)
+        # The database is not initialised.  This is a bit dirty.  This occurs
+        # when running tests.
         pass
     except sqlalchemy.exc.InternalError:
         # The database is not initialised.  Travis hits this

ckan/controllers/group.py

@@ -181,6 +181,9 @@ def read(self, id, limit=20):
         q = c.q = request.params.get('q', '')
 
         try:
+            # Do not query for the group datasets when dictizing, as they will
+            # be ignored and get requested on the controller anyway
+            context['include_datasets'] = False
             c.group_dict = self._action('group_show')(context, data_dict)
             c.group = context['group']
         except NotFound:
@@ -330,6 +333,7 @@ def pager_url(q=None, page=None):
                 items_per_page=limit
             )
 
+            c.group_dict['package_count'] = query['count']
             c.facets = query['facets']
             maintain.deprecate_context_item('facets',
                                             'Use `c.search_facets` instead.')
@@ -372,6 +376,9 @@ def bulk_process(self, id):
         data_dict = {'id': id}
 
         try:
+            # Do not query for the group datasets when dictizing, as they will
+            # be ignored and get requested on the controller anyway
+            context['include_datasets'] = False
             c.group_dict = self._action('group_show')(context, data_dict)
             c.group = context['group']
         except NotFound:

ckan/lib/cli.py

@@ -1322,7 +1322,7 @@ class CreateTestDataCommand(CkanCommand):
                                     translations of terms
     create-test-data vocabs       - annakerenina, warandpeace, and some test
                                     vocabularies
-
+    create-test-data hierarchy    - hierarchy of groups
     '''
     summary = __doc__.split('\n')[0]
     usage = __doc__
@@ -1332,6 +1332,7 @@ class CreateTestDataCommand(CkanCommand):
     def command(self):
         self._load_config()
         self._setup_app()
+        from ckan import plugins
         from create_test_data import CreateTestData
 
         if self.args:
@@ -1356,6 +1357,8 @@ def command(self):
             CreateTestData.create_translations_test_data()
         elif cmd == 'vocabs':
             CreateTestData.create_vocabs_test_data()
+        elif cmd == 'hierarchy':
+            CreateTestData.create_group_hierarchy_test_data()
         else:
             print 'Command %s not recognized' % cmd
             raise NotImplementedError

ckan/lib/create_test_data.py

@@ -38,6 +38,12 @@ def create_family_test_data(cls, extra_users=[]):
                               relationships=family_relationships,
                               extra_user_names=extra_users)
 
+    @classmethod
+    def create_group_hierarchy_test_data(cls, extra_users=[]):
+        cls.create_users(group_hierarchy_users)
+        cls.create_groups(group_hierarchy_groups)
+        cls.create_arbitrary(group_hierarchy_datasets)
+
     @classmethod
     def create_test_user(cls):
         tester = model.User.by_name(u'tester')
@@ -215,18 +221,26 @@ def create_arbitrary(cls, package_dicts, relationships=[],
                             group = model.Group.by_name(unicode(group_name))
                             if not group:
                                 if not group_name in new_groups:
-                                    group = model.Group(name=unicode(group_name))
+                                    group = model.Group(name=
+                                                        unicode(group_name))
                                     model.Session.add(group)
                                     new_group_names.add(group_name)
                                     new_groups[group_name] = group
                                 else:
-                                    # If adding multiple packages with the same group name,
-                                    # model.Group.by_name will not find the group as the
-                                    # session has not yet been committed at this point.
-                                    # Fetch from the new_groups dict instead.
+                                    # If adding multiple packages with the same
+                                    # group name, model.Group.by_name will not
+                                    # find the group as the session has not yet
+                                    # been committed at this point.  Fetch from
+                                    # the new_groups dict instead.
                                     group = new_groups[group_name]
-                            member = model.Member(group=group, table_id=pkg.id, table_name='package')
+                            capacity = 'organization' if group.is_organization\
+                                       else 'public'
+                            member = model.Member(group=group, table_id=pkg.id,
+                                                  table_name='package',
+                                                  capacity=capacity)
                             model.Session.add(member)
+                            if group.is_organization:
+                                pkg.owner_org = group.id
                     elif attr == 'license':
                         pkg.license_id = val
                     elif attr == 'license_id':
@@ -315,38 +329,67 @@ def pkg(pkg_name):
     @classmethod
     def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""):
         '''A more featured interface for creating groups.
-        All group fields can be filled, packages added and they can
-        have an admin user.'''
+        All group fields can be filled, packages added, can have
+        an admin user and be a member of other groups.'''
         rev = model.repo.new_revision()
-        # same name as user we create below
         rev.author = cls.author
         if admin_user_name:
             admin_users = [model.User.by_name(admin_user_name)]
         else:
             admin_users = []
         assert isinstance(group_dicts, (list, tuple))
-        group_attributes = set(('name', 'title', 'description', 'parent_id'))
+        group_attributes = set(('name', 'title', 'description', 'parent_id',
+                                'type', 'is_organization'))
         for group_dict in group_dicts:
-            if model.Group.by_name(group_dict['name']):
-                log.warning('Cannot create group "%s" as it already exists.' % \
-                                (group_dict['name']))
+            if model.Group.by_name(unicode(group_dict['name'])):
+                log.warning('Cannot create group "%s" as it already exists.' %
+                            group_dict['name'])
                 continue
             pkg_names = group_dict.pop('packages', [])
             group = model.Group(name=unicode(group_dict['name']))
             group.type = auth_profile or 'group'
             for key in group_dict:
                 if key in group_attributes:
                     setattr(group, key, group_dict[key])
-                else:
+                elif key not in ('admins', 'editors', 'parent'):
                     group.extras[key] = group_dict[key]
             assert isinstance(pkg_names, (list, tuple))
             for pkg_name in pkg_names:
                 pkg = model.Package.by_name(unicode(pkg_name))
                 assert pkg, pkg_name
-                member = model.Member(group=group, table_id=pkg.id, table_name='package')
+                member = model.Member(group=group, table_id=pkg.id,
+                                      table_name='package')
                 model.Session.add(member)
             model.Session.add(group)
-            model.setup_default_user_roles(group, admin_users)
+            admins = [model.User.by_name(user_name)
+                      for user_name in group_dict.get('admins', [])] + \
+                     admin_users
+            for admin in admins:
+                member = model.Member(group=group, table_id=admin.id,
+                                      table_name='user', capacity='admin')
+                model.Session.add(member)
+            editors = [model.User.by_name(user_name)
+                       for user_name in group_dict.get('editors', [])]
+            for editor in editors:
+                member = model.Member(group=group, table_id=editor.id,
+                                      table_name='user', capacity='editor')
+                model.Session.add(member)
+            # Need to commit the current Group for two reasons:
+            # 1. It might have a parent, and the Member will need the Group.id
+            #    value allocated on commit.
+            # 2. The next Group created may have this Group as a parent so
+            #    creation of the Member needs to refer to this one.
+            model.Session.commit()
+            rev = model.repo.new_revision()
+            rev.author = cls.author
+            # add it to a parent's group
+            if 'parent' in group_dict:
+                parent = model.Group.by_name(unicode(group_dict['parent']))
+                assert parent, group_dict['parent']
+                member = model.Member(group=group, table_id=parent.id,
+                                      table_name='group', capacity='parent')
+                model.Session.add(member)
+            #model.setup_default_user_roles(group, admin_users)
             cls.group_names.add(group_dict['name'])
         model.repo.commit_and_remove()
 
@@ -362,7 +405,8 @@ def create(cls, auth_profile="", package_type=None):
  * Associated tags, etc etc
 '''
         if auth_profile == "publisher":
-            organization_group = model.Group(name=u"organization_group", type="organization")
+            organization_group = model.Group(name=u"organization_group",
+                                             type="organization")
 
         cls.pkg_names = [u'annakarenina', u'warandpeace']
         pkg1 = model.Package(name=cls.pkg_names[0], type=package_type)
@@ -483,11 +527,12 @@ def create(cls, auth_profile="", package_type=None):
         roger = model.Group.by_name(u'roger')
         model.setup_default_user_roles(david, [russianfan])
         model.setup_default_user_roles(roger, [russianfan])
-        model.add_user_to_role(visitor, model.Role.ADMIN, roger)
 
+        # in new_authz you can't give a visitor permissions to a
+        # group it seems, so this is a bit meaningless
+        model.add_user_to_role(visitor, model.Role.ADMIN, roger)
         model.repo.commit_and_remove()
 
-
     # method used in DGU and all good tests elsewhere
     @classmethod
     def create_users(cls, user_dicts):
@@ -501,9 +546,11 @@ def create_users(cls, user_dicts):
 
     @classmethod
     def _create_user_without_commit(cls, name='', **user_dict):
-        if model.User.by_name(name) or (user_dict.get('open_id') and model.User.by_openid(user_dict.get('openid'))):
-            log.warning('Cannot create user "%s" as it already exists.' % \
-                            (name or user_dict['name']))
+        if model.User.by_name(name) or \
+                (user_dict.get('open_id') and
+                 model.User.by_openid(user_dict.get('openid'))):
+            log.warning('Cannot create user "%s" as it already exists.' %
+                        name or user_dict['name'])
             return
         # User objects are not revisioned so no need to create a revision
         user_ref = name or user_dict['openid']
@@ -826,6 +873,65 @@ def make_some_vocab_tags(cls):
      }
     ]
 
+group_hierarchy_groups = [
+    {'name': 'department-of-health',
+     'title': 'Department of Health',
+     'contact-email': 'contact@doh.gov.uk',
+     'type': 'organization',
+     'is_organization': True
+     },
+    {'name': 'food-standards-agency',
+     'title': 'Food Standards Agency',
+     'contact-email': 'contact@fsa.gov.uk',
+     'parent': 'department-of-health',
+     'type': 'organization',
+     'is_organization': True},
+    {'name': 'national-health-service',
+     'title': 'National Health Service',
+     'contact-email': 'contact@nhs.gov.uk',
+     'parent': 'department-of-health',
+     'type': 'organization',
+     'is_organization': True,
+     'editors': ['nhseditor'],
+     'admins': ['nhsadmin']},
+    {'name': 'nhs-wirral-ccg',
+     'title': 'NHS Wirral CCG',
+     'contact-email': 'contact@wirral.nhs.gov.uk',
+     'parent': 'national-health-service',
+     'type': 'organization',
+     'is_organization': True,
+     'editors': ['wirraleditor'],
+     'admins': ['wirraladmin']},
+    {'name': 'nhs-southwark-ccg',
+     'title': 'NHS Southwark CCG',
+     'contact-email': 'contact@southwark.nhs.gov.uk',
+     'parent': 'national-health-service',
+     'type': 'organization',
+     'is_organization': True},
+    {'name': 'cabinet-office',
+     'title': 'Cabinet Office',
+     'contact-email': 'contact@cabinet-office.gov.uk',
+     'type': 'organization',
+     'is_organization': True},
+    ]
+
+group_hierarchy_datasets = [
+    {'name': 'doh-spend', 'title': 'Department of Health Spend Data',
+     'groups': ['department-of-health']},
+    {'name': 'nhs-spend', 'title': 'NHS Spend Data',
+     'groups': ['national-health-service']},
+    {'name': 'wirral-spend', 'title': 'Wirral Spend Data',
+     'groups': ['nhs-wirral-ccg']},
+    {'name': 'southwark-spend', 'title': 'Southwark Spend Data',
+     'groups': ['nhs-southwark-ccg']},
+    ]
+
+group_hierarchy_users = [{'name': 'nhsadmin', 'password': 'pass'},
+                         {'name': 'nhseditor', 'password': 'pass'},
+                         {'name': 'wirraladmin', 'password': 'pass'},
+                         {'name': 'wirraleditor', 'password': 'pass'},
+                         ]
+
 # Some test terms and translations.
 terms = ('A Novel By Tolstoy',
     'Index of the novel',

ckan/lib/dictization/model_dictize.py

@@ -348,7 +348,6 @@ def _get_members(context, group, member_type):
 
 
 def group_dictize(group, context):
-    model = context['model']
     result_dict = d.table_dictize(group, context)
 
     result_dict['display_name'] = group.display_name
@@ -358,16 +357,32 @@ def group_dictize(group, context):
 
     context['with_capacity'] = True
 
-    result_dict['packages'] = d.obj_list_dictize(
-        _get_members(context, group, 'packages'),
-        context)
+    include_datasets = context.get('include_datasets', True)
+
+    q = {
+        'facet': 'false',
+        'rows': 0,
+    }
 
-    query = search.PackageSearchQuery()
     if group.is_organization:
-        q = {'q': 'owner_org:"%s" +capacity:public' % group.id, 'rows': 1}
+        q['fq'] = 'owner_org:"{0}"'.format(group.id)
     else:
-        q = {'q': 'groups:"%s" +capacity:public' % group.name, 'rows': 1}
-    result_dict['package_count'] = query.run(q)['count']
+        q['fq'] = 'groups:"{0}"'.format(group.name)
+
+    is_group_member = (context.get('user') and
+         new_authz.has_user_permission_for_group_or_org(group.id, context.get('user'), 'read'))
+    if is_group_member:
+        context['ignore_capacity_check'] = True
+
+    if include_datasets:
+        q['rows'] = 1000    # Only the first 1000 datasets are returned
+
+    search_results = logic.get_action('package_search')(context, q)
+
+    if include_datasets:
+        result_dict['packages'] = search_results['results']
+
+    result_dict['package_count'] = search_results['count']
 
     result_dict['tags'] = tag_list_dictize(
         _get_members(context, group, 'tags'),

ckan/lib/dictization/model_save.py

@@ -342,7 +342,11 @@ def group_member_save(context, group_dict, member_table_name):
     entities = {}
     Member = model.Member
 
-    ModelClass = getattr(model, member_table_name[:-1].capitalize())
+    classname = member_table_name[:-1].capitalize()
+    if classname == 'Organization':
+        # Organizations use the model.Group class
+        classname = 'Group'
+    ModelClass = getattr(model, classname)
 
     for entity_dict in entity_list:
         name_or_id = entity_dict.get('id') or entity_dict.get('name')

ckan/lib/plugins.py

@@ -115,7 +115,7 @@ def register_group_plugins(map):
     """
     Register the various IGroupForm instances.
 
-    This method will setup the mappings between package types and the
+    This method will setup the mappings between group types and the
     registered IGroupForm instances. If it's called more than once an
     exception will be raised.
     """
@@ -160,7 +160,7 @@ def register_group_plugins(map):
 
             if group_type in _group_plugins:
                 raise ValueError, "An existing IGroupForm is "\
-                                  "already associated with the package type "\
+                                  "already associated with the group type "\
                                   "'%s'" % group_type
             _group_plugins[group_type] = plugin