Skip to content

Commit

Permalink
Merge branch 'master' of github.com:okfn/ckan into 847-new-theming-docs
Browse files Browse the repository at this point in the history
  • Loading branch information
Sean Hammond committed Dec 13, 2013
2 parents 839b5c8 + 6213f0d commit 3e1847b
Show file tree
Hide file tree
Showing 60 changed files with 3,740 additions and 425 deletions.
4 changes: 4 additions & 0 deletions CHANGELOG.rst
View file
Expand Up @@ -19,6 +19,10 @@ API changes and deprecations:
* The edit() and after_update() methods of IPackageController plugins are now
called when updating a resource using the web frontend or the
resource_update API action [#1052]
* package_search now returns results with custom schemas applied like
package_show, a use_default_schema parameter was added to request the
old behaviour, this change may affect customized search result templates
(#1255)

v2.1.1 2013-11-8
================
Expand Down
1 change: 1 addition & 0 deletions ckan/config/deployment.ini_tmpl
View file
Expand Up @@ -67,6 +67,7 @@ ckan.auth.user_delete_groups = true
ckan.auth.user_delete_organizations = true
ckan.auth.create_user_via_api = false
ckan.auth.create_user_via_web = true
ckan.auth.roles_that_cascade_to_sub_groups = admin


## Search Settings
Expand Down
6 changes: 4 additions & 2 deletions ckan/config/environment.py
View file
Expand Up @@ -367,8 +367,10 @@ def template_loaded(template):
# Here we create the site user if they are not already in the database
try:
logic.get_action('get_site_user')({'ignore_auth': True}, None)
except sqlalchemy.exc.ProgrammingError:
# The database is not initialised. This is a bit dirty.
except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError):
# (ProgrammingError for Postgres, OperationalError for SQLite)
# The database is not initialised. This is a bit dirty. This occurs
# when running tests.
pass
except sqlalchemy.exc.InternalError:
# The database is not initialised. Travis hits this
Expand Down
10 changes: 10 additions & 0 deletions ckan/controllers/group.py
View file
Expand Up @@ -168,6 +168,9 @@ def index(self):

def read(self, id, limit=20):
group_type = self._get_group_type(id.split('@')[0])
if group_type != self.group_type:
abort(404, _('Incorrect group type'))

context = {'model': model, 'session': model.Session,
'user': c.user or c.author,
'schema': self._db_to_form_schema(group_type=group_type),
Expand All @@ -178,6 +181,9 @@ def read(self, id, limit=20):
q = c.q = request.params.get('q', '')

try:
# Do not query for the group datasets when dictizing, as they will
# be ignored and get requested on the controller anyway
context['include_datasets'] = False
c.group_dict = self._action('group_show')(context, data_dict)
c.group = context['group']
except NotFound:
Expand Down Expand Up @@ -327,6 +333,7 @@ def pager_url(q=None, page=None):
items_per_page=limit
)

c.group_dict['package_count'] = query['count']
c.facets = query['facets']
maintain.deprecate_context_item('facets',
'Use `c.search_facets` instead.')
Expand Down Expand Up @@ -369,6 +376,9 @@ def bulk_process(self, id):
data_dict = {'id': id}

try:
# Do not query for the group datasets when dictizing, as they will
# be ignored and get requested on the controller anyway
context['include_datasets'] = False
c.group_dict = self._action('group_show')(context, data_dict)
c.group = context['group']
except NotFound:
Expand Down
1 change: 0 additions & 1 deletion ckan/lib/accept.py
View file
Expand Up @@ -12,7 +12,6 @@
# Name : ContentType, Is Markup?, Extension
"text/html": ("text/html; charset=utf-8", True, 'html'),
"text/n3": ("text/n3; charset=utf-8", False, 'n3'),
"text/plain": ("text/plain; charset=utf-8", False, 'txt'),
"application/rdf+xml": ("application/rdf+xml; charset=utf-8", True, 'rdf'),
}
accept_by_extension = {
Expand Down
5 changes: 4 additions & 1 deletion ckan/lib/cli.py
View file
Expand Up @@ -1322,7 +1322,7 @@ class CreateTestDataCommand(CkanCommand):
translations of terms
create-test-data vocabs - annakerenina, warandpeace, and some test
vocabularies
create-test-data hierarchy - hierarchy of groups
'''
summary = __doc__.split('\n')[0]
usage = __doc__
Expand All @@ -1332,6 +1332,7 @@ class CreateTestDataCommand(CkanCommand):
def command(self):
self._load_config()
self._setup_app()
from ckan import plugins
from create_test_data import CreateTestData

if self.args:
Expand All @@ -1356,6 +1357,8 @@ def command(self):
CreateTestData.create_translations_test_data()
elif cmd == 'vocabs':
CreateTestData.create_vocabs_test_data()
elif cmd == 'hierarchy':
CreateTestData.create_group_hierarchy_test_data()
else:
print 'Command %s not recognized' % cmd
raise NotImplementedError
Expand Down
150 changes: 128 additions & 22 deletions ckan/lib/create_test_data.py
View file
Expand Up @@ -38,6 +38,12 @@ def create_family_test_data(cls, extra_users=[]):
relationships=family_relationships,
extra_user_names=extra_users)

@classmethod
def create_group_hierarchy_test_data(cls, extra_users=[]):
cls.create_users(group_hierarchy_users)
cls.create_groups(group_hierarchy_groups)
cls.create_arbitrary(group_hierarchy_datasets)

@classmethod
def create_test_user(cls):
tester = model.User.by_name(u'tester')
Expand Down Expand Up @@ -215,18 +221,26 @@ def create_arbitrary(cls, package_dicts, relationships=[],
group = model.Group.by_name(unicode(group_name))
if not group:
if not group_name in new_groups:
group = model.Group(name=unicode(group_name))
group = model.Group(name=
unicode(group_name))
model.Session.add(group)
new_group_names.add(group_name)
new_groups[group_name] = group
else:
# If adding multiple packages with the same group name,
# model.Group.by_name will not find the group as the
# session has not yet been committed at this point.
# Fetch from the new_groups dict instead.
# If adding multiple packages with the same
# group name, model.Group.by_name will not
# find the group as the session has not yet
# been committed at this point. Fetch from
# the new_groups dict instead.
group = new_groups[group_name]
member = model.Member(group=group, table_id=pkg.id, table_name='package')
capacity = 'organization' if group.is_organization\
else 'public'
member = model.Member(group=group, table_id=pkg.id,
table_name='package',
capacity=capacity)
model.Session.add(member)
if group.is_organization:
pkg.owner_org = group.id
elif attr == 'license':
pkg.license_id = val
elif attr == 'license_id':
Expand Down Expand Up @@ -315,38 +329,67 @@ def pkg(pkg_name):
@classmethod
def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""):
'''A more featured interface for creating groups.
All group fields can be filled, packages added and they can
have an admin user.'''
All group fields can be filled, packages added, can have
an admin user and be a member of other groups.'''
rev = model.repo.new_revision()
# same name as user we create below
rev.author = cls.author
if admin_user_name:
admin_users = [model.User.by_name(admin_user_name)]
else:
admin_users = []
assert isinstance(group_dicts, (list, tuple))
group_attributes = set(('name', 'title', 'description', 'parent_id'))
group_attributes = set(('name', 'title', 'description', 'parent_id',
'type', 'is_organization'))
for group_dict in group_dicts:
if model.Group.by_name(group_dict['name']):
log.warning('Cannot create group "%s" as it already exists.' % \
(group_dict['name']))
if model.Group.by_name(unicode(group_dict['name'])):
log.warning('Cannot create group "%s" as it already exists.' %
group_dict['name'])
continue
pkg_names = group_dict.pop('packages', [])
group = model.Group(name=unicode(group_dict['name']))
group.type = auth_profile or 'group'
for key in group_dict:
if key in group_attributes:
setattr(group, key, group_dict[key])
else:
elif key not in ('admins', 'editors', 'parent'):
group.extras[key] = group_dict[key]
assert isinstance(pkg_names, (list, tuple))
for pkg_name in pkg_names:
pkg = model.Package.by_name(unicode(pkg_name))
assert pkg, pkg_name
member = model.Member(group=group, table_id=pkg.id, table_name='package')
member = model.Member(group=group, table_id=pkg.id,
table_name='package')
model.Session.add(member)
model.Session.add(group)
model.setup_default_user_roles(group, admin_users)
admins = [model.User.by_name(user_name)
for user_name in group_dict.get('admins', [])] + \
admin_users
for admin in admins:
member = model.Member(group=group, table_id=admin.id,
table_name='user', capacity='admin')
model.Session.add(member)
editors = [model.User.by_name(user_name)
for user_name in group_dict.get('editors', [])]
for editor in editors:
member = model.Member(group=group, table_id=editor.id,
table_name='user', capacity='editor')
model.Session.add(member)
# Need to commit the current Group for two reasons:
# 1. It might have a parent, and the Member will need the Group.id
# value allocated on commit.
# 2. The next Group created may have this Group as a parent so
# creation of the Member needs to refer to this one.
model.Session.commit()
rev = model.repo.new_revision()
rev.author = cls.author
# add it to a parent's group
if 'parent' in group_dict:
parent = model.Group.by_name(unicode(group_dict['parent']))
assert parent, group_dict['parent']
member = model.Member(group=group, table_id=parent.id,
table_name='group', capacity='parent')
model.Session.add(member)
#model.setup_default_user_roles(group, admin_users)
cls.group_names.add(group_dict['name'])
model.repo.commit_and_remove()

Expand All @@ -362,7 +405,8 @@ def create(cls, auth_profile="", package_type=None):
* Associated tags, etc etc
'''
if auth_profile == "publisher":
organization_group = model.Group(name=u"organization_group", type="organization")
organization_group = model.Group(name=u"organization_group",
type="organization")

cls.pkg_names = [u'annakarenina', u'warandpeace']
pkg1 = model.Package(name=cls.pkg_names[0], type=package_type)
Expand Down Expand Up @@ -483,11 +527,12 @@ def create(cls, auth_profile="", package_type=None):
roger = model.Group.by_name(u'roger')
model.setup_default_user_roles(david, [russianfan])
model.setup_default_user_roles(roger, [russianfan])
model.add_user_to_role(visitor, model.Role.ADMIN, roger)

# in new_authz you can't give a visitor permissions to a
# group it seems, so this is a bit meaningless
model.add_user_to_role(visitor, model.Role.ADMIN, roger)
model.repo.commit_and_remove()


# method used in DGU and all good tests elsewhere
@classmethod
def create_users(cls, user_dicts):
Expand All @@ -501,9 +546,11 @@ def create_users(cls, user_dicts):

@classmethod
def _create_user_without_commit(cls, name='', **user_dict):
if model.User.by_name(name) or (user_dict.get('open_id') and model.User.by_openid(user_dict.get('openid'))):
log.warning('Cannot create user "%s" as it already exists.' % \
(name or user_dict['name']))
if model.User.by_name(name) or \
(user_dict.get('open_id') and
model.User.by_openid(user_dict.get('openid'))):
log.warning('Cannot create user "%s" as it already exists.' %
name or user_dict['name'])
return
# User objects are not revisioned so no need to create a revision
user_ref = name or user_dict['openid']
Expand Down Expand Up @@ -826,6 +873,65 @@ def make_some_vocab_tags(cls):
}
]

group_hierarchy_groups = [
{'name': 'department-of-health',
'title': 'Department of Health',
'contact-email': 'contact@doh.gov.uk',
'type': 'organization',
'is_organization': True
},
{'name': 'food-standards-agency',
'title': 'Food Standards Agency',
'contact-email': 'contact@fsa.gov.uk',
'parent': 'department-of-health',
'type': 'organization',
'is_organization': True},
{'name': 'national-health-service',
'title': 'National Health Service',
'contact-email': 'contact@nhs.gov.uk',
'parent': 'department-of-health',
'type': 'organization',
'is_organization': True,
'editors': ['nhseditor'],
'admins': ['nhsadmin']},
{'name': 'nhs-wirral-ccg',
'title': 'NHS Wirral CCG',
'contact-email': 'contact@wirral.nhs.gov.uk',
'parent': 'national-health-service',
'type': 'organization',
'is_organization': True,
'editors': ['wirraleditor'],
'admins': ['wirraladmin']},
{'name': 'nhs-southwark-ccg',
'title': 'NHS Southwark CCG',
'contact-email': 'contact@southwark.nhs.gov.uk',
'parent': 'national-health-service',
'type': 'organization',
'is_organization': True},
{'name': 'cabinet-office',
'title': 'Cabinet Office',
'contact-email': 'contact@cabinet-office.gov.uk',
'type': 'organization',
'is_organization': True},
]

group_hierarchy_datasets = [
{'name': 'doh-spend', 'title': 'Department of Health Spend Data',
'groups': ['department-of-health']},
{'name': 'nhs-spend', 'title': 'NHS Spend Data',
'groups': ['national-health-service']},
{'name': 'wirral-spend', 'title': 'Wirral Spend Data',
'groups': ['nhs-wirral-ccg']},
{'name': 'southwark-spend', 'title': 'Southwark Spend Data',
'groups': ['nhs-southwark-ccg']},
]

group_hierarchy_users = [{'name': 'nhsadmin', 'password': 'pass'},
{'name': 'nhseditor', 'password': 'pass'},
{'name': 'wirraladmin', 'password': 'pass'},
{'name': 'wirraleditor', 'password': 'pass'},
]

# Some test terms and translations.
terms = ('A Novel By Tolstoy',
'Index of the novel',
Expand Down
31 changes: 23 additions & 8 deletions ckan/lib/dictization/model_dictize.py
View file
Expand Up @@ -348,7 +348,6 @@ def _get_members(context, group, member_type):


def group_dictize(group, context):
model = context['model']
result_dict = d.table_dictize(group, context)

result_dict['display_name'] = group.display_name
Expand All @@ -358,16 +357,32 @@ def group_dictize(group, context):

context['with_capacity'] = True

result_dict['packages'] = d.obj_list_dictize(
_get_members(context, group, 'packages'),
context)
include_datasets = context.get('include_datasets', True)

q = {
'facet': 'false',
'rows': 0,
}

query = search.PackageSearchQuery()
if group.is_organization:
q = {'q': 'owner_org:"%s" +capacity:public' % group.id, 'rows': 1}
q['fq'] = 'owner_org:"{0}"'.format(group.id)
else:
q = {'q': 'groups:"%s" +capacity:public' % group.name, 'rows': 1}
result_dict['package_count'] = query.run(q)['count']
q['fq'] = 'groups:"{0}"'.format(group.name)

is_group_member = (context.get('user') and
new_authz.has_user_permission_for_group_or_org(group.id, context.get('user'), 'read'))
if is_group_member:
context['ignore_capacity_check'] = True

if include_datasets:
q['rows'] = 1000 # Only the first 1000 datasets are returned

search_results = logic.get_action('package_search')(context, q)

if include_datasets:
result_dict['packages'] = search_results['results']

result_dict['package_count'] = search_results['count']

result_dict['tags'] = tag_list_dictize(
_get_members(context, group, 'tags'),
Expand Down
6 changes: 5 additions & 1 deletion ckan/lib/dictization/model_save.py
View file
Expand Up @@ -342,7 +342,11 @@ def group_member_save(context, group_dict, member_table_name):
entities = {}
Member = model.Member

ModelClass = getattr(model, member_table_name[:-1].capitalize())
classname = member_table_name[:-1].capitalize()
if classname == 'Organization':
# Organizations use the model.Group class
classname = 'Group'
ModelClass = getattr(model, classname)

for entity_dict in entity_list:
name_or_id = entity_dict.get('id') or entity_dict.get('name')
Expand Down

0 comments on commit 3e1847b

Please sign in to comment.