From 35fc3ed37b600570b5a294cf9621c1400bd48bf1 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 25 May 2012 16:50:17 +0100 Subject: [PATCH 001/395] [xs] Helpful SOLR hint. --- doc/solr-setup.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/solr-setup.rst b/doc/solr-setup.rst index 395da67e0a9..c7c2f53ee2d 100644 --- a/doc/solr-setup.rst +++ b/doc/solr-setup.rst @@ -192,6 +192,18 @@ Some problems that can be found during the install: ${dataDir} +* When running Solr it says `Unable to find a javac compiler; com.sun.tools.javac.Main is not on the classpath. Perhaps JAVA_HOME does not point to the JDK.` + + See the note above about JAVA_HOME. Alternatively you may not have installed the JDK. Check by seeing if javac is installed:: + + which javac + + If it isn't do:: + + sudo apt-get install openjdk-6-jdk + + and restart SOLR. + Handling changes in the CKAN schema ----------------------------------- From 04ca4191673010e86ac0d77991cb7967fe6c58ae Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 15 Jun 2012 13:45:46 +0100 Subject: [PATCH 002/395] Fix minor bug that caused create_users to not commit changes. --- ckan/lib/create_test_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 53743e6c2cc..1b8d3b601a2 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -537,6 +537,7 @@ def _create_user_without_commit(cls, name='', **user_dict): user = model.User(name=unicode(name), **user_dict) model.Session.add(user) cls.user_refs.append(user_ref) + return user @classmethod def create_user(cls, name='', **kwargs): From 6780e91351915bf0a020c93cae67d597c406c348 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 15 Jun 2012 13:46:04 +0100 Subject: [PATCH 003/395] Improve logging in useful places. --- ckan/controllers/package.py | 2 ++ ckan/lib/authenticator.py | 6 ++++++ ckan/lib/base.py | 3 +++ ckan/lib/search/__init__.py | 13 +++++++------ 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 9403a0271e3..a7df2cf1849 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -592,6 +592,8 @@ def _save_new(self, context, package_type=None): def _save_edit(self, name_or_id, context): from ckan.lib.search import SearchIndexError + log.debug('Package save request name: %s POST: %r', + name_or_id, request.POST) try: data_dict = clean_dict(unflatten( tuplize_dict(parse_params(request.POST)))) diff --git a/ckan/lib/authenticator.py b/ckan/lib/authenticator.py index b56711a3427..6f061caad91 100644 --- a/ckan/lib/authenticator.py +++ b/ckan/lib/authenticator.py @@ -1,8 +1,12 @@ +import logging + from zope.interface import implements from repoze.who.interfaces import IAuthenticator from ckan.model import User, Session +log = logging.getLogger(__name__) + class OpenIDAuthenticator(object): implements(IAuthenticator) @@ -25,8 +29,10 @@ def authenticate(self, environ, identity): return None user = User.by_name(identity.get('login')) if user is None: + log.debug('Login failed - username %r not found', identity.get('login')) return None if user.validate_password(identity.get('password')): return user.name + log.debug('Login as %r failed - password not valid', identity.get('login')) return None diff --git a/ckan/lib/base.py b/ckan/lib/base.py index f2cc447fdea..52604d48e7f 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -29,6 +29,8 @@ from ckan.lib.helpers import json import ckan.model as model +log = logging.getLogger(__name__) + PAGINATE_ITEMS_PER_PAGE = 50 APIKEY_HEADER_NAME_KEY = 'apikey_header_name' @@ -139,6 +141,7 @@ def render_template(): response.headers["Cache-Control"] = "private" # Prevent any further rendering from being cached. request.environ['__no_cache__'] = True + log.debug('Template cache-control: %s' % response.headers["Cache-Control"]) # Render Time :) try: diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index abc78b798fd..b68e7850c47 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -128,11 +128,11 @@ def rebuild(package_id=None,only_missing=False,force=False,refresh=False): If a dataset id is provided, only this dataset will be reindexed. When reindexing all datasets, if only_missing is True, only the datasets not already indexed will be processed. If force equals - True, if an execption is found, the exception will be logged, but + True, if an exception is found, the exception will be logged, but the process will carry on. ''' from ckan import model - log.debug("Rebuilding search index...") + log.info("Rebuilding search index...") package_index = index_for(model.Package) @@ -140,21 +140,22 @@ def rebuild(package_id=None,only_missing=False,force=False,refresh=False): pkg_dict = get_action('package_show')( {'model': model, 'ignore_auth': True, 'validate': False}, {'id': package_id}) + log.info('Indexing just package %r...', pkg_dict['name']) package_index.remove_dict(pkg_dict) package_index.insert_dict(pkg_dict) else: package_ids = [r[0] for r in model.Session.query(model.Package.id).filter(model.Package.state == 'active').all()] if only_missing: - log.debug('Indexing only missing packages...') + log.info('Indexing only missing packages...') package_query = query_for(model.Package) indexed_pkg_ids = set(package_query.get_all_entity_ids(max_results=len(package_ids))) package_ids = set(package_ids) - indexed_pkg_ids # Packages not indexed if len(package_ids) == 0: - log.debug('All datasets are already indexed') + log.info('All datasets are already indexed') return else: - log.debug('Rebuilding the whole index...') + log.info('Rebuilding the whole index...') # When refreshing, the index is not previously cleared if not refresh: package_index.clear() @@ -176,7 +177,7 @@ def rebuild(package_id=None,only_missing=False,force=False,refresh=False): raise model.Session.commit() - log.debug('Finished rebuilding search index.') + log.info('Finished rebuilding search index.') def check(): from ckan import model From 75ba31027650b1ff3af3c8dc16cc11663f515350 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 21 Jun 2012 11:22:36 +0100 Subject: [PATCH 004/395] Comment required, else Toby would just delete the method out of hand. --- ckan/lib/create_test_data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 1b8d3b601a2..7b4a7815359 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -511,6 +511,7 @@ def create(cls, auth_profile="", package_type=None): model.repo.commit_and_remove() + # method used in DGU and all good tests elsewhere @classmethod def create_users(cls, user_dicts): needs_commit = False From 98888d2b5a46945a96b1d4719718c836a6a15e60 Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 22 Jul 2013 17:43:45 +0100 Subject: [PATCH 005/395] [#1038] Correct copyright sign for UTF8 file and cut/paste error. --- ckan/config/environment.py | 2 +- ckan/lib/plugins.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 7a2dd27682d..fd8b46fa98d 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -237,7 +237,7 @@ def template_loaded(template): ''' This code is based on Genshi code - Copyright © 2006-2012 Edgewall Software + Copyright © 2006-2012 Edgewall Software All rights reserved. Redistribution and use in source and binary forms, with or diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index f60b928f434..1bf87ac29cb 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -104,7 +104,7 @@ def register_group_plugins(map): """ Register the various IGroupForm instances. - This method will setup the mappings between package types and the + This method will setup the mappings between group types and the registered IGroupForm instances. If it's called more than once an exception will be raised. """ @@ -149,7 +149,7 @@ def register_group_plugins(map): if group_type in _group_plugins: raise ValueError, "An existing IGroupForm is "\ - "already associated with the package type "\ + "already associated with the group type "\ "'%s'" % group_type _group_plugins[group_type] = plugin From 21cd0ac33023d13127a1e66094334de9d53b8b24 Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 22 Jul 2013 17:51:18 +0100 Subject: [PATCH 006/395] [#1038] Fix to allow sqlite testing - for until the tests are overhaulled. --- ckan/config/environment.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index fd8b46fa98d..49348789369 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -353,6 +353,8 @@ def genshi_lookup_attr(cls, obj, key): # Here we create the site user if they are not already in the database try: logic.get_action('get_site_user')({'ignore_auth': True}, None) - except sqlalchemy.exc.ProgrammingError: - # The database is not initialised. This is a bit dirty. + except (sqlalchemy.exc.ProgrammingError, sqlalchemy.exc.OperationalError): + # (ProgrammingError for Postgres, OperationalError for SQLite) + # The database is not initialised. This is a bit dirty. This occurs + # when running tests. pass From 96f0a3e775e14207cd1eec3b82f0e05a1f5a48ed Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 22 Jul 2013 17:52:48 +0100 Subject: [PATCH 007/395] [#1038] Test fixtures for organization hierarchy. --- ckan/lib/cli.py | 4 +- ckan/lib/create_test_data.py | 81 +++++++++++++++++++++++++++++++++--- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index bf4654e2e09..0acae3550bb 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -1273,7 +1273,7 @@ class CreateTestDataCommand(CkanCommand): translations of terms create-test-data vocabs - annakerenina, warandpeace, and some test vocabularies - + create-test-data hierarchy - hierarchy of groups ''' summary = __doc__.split('\n')[0] usage = __doc__ @@ -1309,6 +1309,8 @@ def command(self): CreateTestData.create_translations_test_data() elif cmd == 'vocabs': CreateTestData.create_vocabs_test_data() + elif cmd == 'hierarchy': + CreateTestData.create_group_hierarchy_test_data() else: print 'Command %s not recognized' % cmd raise NotImplementedError diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 3edf865c7c7..f464907d745 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -38,6 +38,12 @@ def create_family_test_data(cls, extra_users=[]): relationships=family_relationships, extra_user_names=extra_users) + @classmethod + def create_group_hierarchy_test_data(cls, extra_users=[]): + cls.create_groups(group_hierarchy_groups) + cls.create_arbitrary(group_hierarchy_datasets, + extra_user_names=group_hierarchy_users) + @classmethod def create_test_user(cls): tester = model.User.by_name(u'tester') @@ -315,10 +321,9 @@ def pkg(pkg_name): @classmethod def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): '''A more featured interface for creating groups. - All group fields can be filled, packages added and they can - have an admin user.''' + All group fields can be filled, packages added, can have + an admin user and be a member of other groups.''' rev = model.repo.new_revision() - # same name as user we create below rev.author = cls.author if admin_user_name: admin_users = [model.User.by_name(admin_user_name)] @@ -327,7 +332,7 @@ def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): assert isinstance(group_dicts, (list, tuple)) group_attributes = set(('name', 'title', 'description', 'parent_id')) for group_dict in group_dicts: - if model.Group.by_name(group_dict['name']): + if model.Group.by_name(unicode(group_dict['name'])): log.warning('Cannot create group "%s" as it already exists.' % \ (group_dict['name'])) continue @@ -346,7 +351,35 @@ def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): member = model.Member(group=group, table_id=pkg.id, table_name='package') model.Session.add(member) model.Session.add(group) - model.setup_default_user_roles(group, admin_users) + admins = [model.User.by_name(user_name) \ + for user_name in group_dict.get('admins', [])] \ + + admin_users + for admin in admins: + member = model.Member(group=group, table_id=user.id, + table_name='user', capacity='admin') + model.Session.add(member) + editors = [model.User.by_name(user_name) \ + for user_name in group_dict.get('editors', [])] + for editor in editors: + member = model.Member(group=group, table_id=user.id, + table_name='user', capacity='editor') + model.Session.add(member) + # Need to commit the current Group for two reasons: + # 1. It might have a parent, and the Member will need the Group.id + # value allocated on commit. + # 2. The next Group created may have this Group as a parent so + # creation of the Member needs to refer to this one. + model.Session.commit() + rev = model.repo.new_revision() + rev.author = cls.author + # add it to a parent's group + if 'parent' in group_dict: + parent = model.Group.by_name(unicode(group_dict['parent'])) + assert parent, group_dict['parent'] + member = model.Member(group=parent, table_id=group.id, + table_name='group', capacity='parent') + model.Session.add(member) + #model.setup_default_user_roles(group, admin_users) cls.group_names.add(group_dict['name']) model.repo.commit_and_remove() @@ -825,6 +858,44 @@ def make_some_vocab_tags(cls): } ] +group_hierarchy_groups = [ + {'name': 'department-of-health', + 'title': 'Department of Health', + 'contact-email': 'contact@doh.gov.uk'}, + {'name': 'food-standards-agency', + 'title': 'Food Standards Agency', + 'contact-email': 'contact@fsa.gov.uk', + 'parent': 'department-of-health'}, + {'name': 'national-health-service', + 'title': 'National Health Service', + 'contact-email': 'contact@nhs.gov.uk', + 'parent': 'department-of-health'}, + {'name': 'nhs-wirral-ccg', + 'title': 'NHS Wirral CCG', + 'contact-email': 'contact@wirral.nhs.gov.uk', + 'parent': 'national-health-service'}, + {'name': 'nhs-southwark-ccg', + 'title': 'NHS Southwark CCG', + 'contact-email': 'contact@southwark.nhs.gov.uk', + 'parent': 'national-health-service'}, + {'name': 'cabinet-office', + 'title': 'Cabinet Office', + 'contact-email': 'contact@cabinet-office.gov.uk'}, + ] + +group_hierarchy_datasets = [ + {'name': 'doh-spend', 'title': 'Department of Health Spend Data', + 'groups': ['department-of-health']}, + {'name': 'nhs-spend', 'title': 'NHS Spend Data', + 'groups': ['national-health-service']}, + {'name': 'wirral-spend', 'title': 'Wirral Spend Data', + 'groups': ['nhs-wirral-ccg']}, + {'name': 'southwark-spend', 'title': 'Southwark Spend Data', + 'groups': ['nhs-southwark-ccg']}, + ] + +group_hierarchy_users = ['nhsadmin', 'nhseditor', 'wirraladmin', 'wirraleditor'] + # Some test terms and translations. terms = ('A Novel By Tolstoy', 'Index of the novel', From fde50fa47123c714e8098e8f7dbf88085da67ddc Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 22 Jul 2013 17:54:56 +0100 Subject: [PATCH 008/395] [#1038] Model methods for organization hierarchy. --- ckan/model/group.py | 111 ++++++++++++++++++++++++++------ ckan/tests/models/test_group.py | 64 +++++++++++++++++- 2 files changed, 153 insertions(+), 22 deletions(-) diff --git a/ckan/model/group.py b/ckan/model/group.py index c16424affee..2cf748bb078 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -87,6 +87,17 @@ def related_packages(self): return meta.Session.query(_package.Package).filter_by( id=self.table_id).all() + def __unicode__(self): + # refer to objects by name, not ID, to help debugging + if self.table_name == 'package': + table_info = 'package=%s' % meta.Session.query(_package.Package).get(self.table_id).name + elif self.table_name == 'group': + table_info = 'group=%s' % meta.Session.query(Group).get(self.table_id).name + else: + table_info = 'table_name=%s table_id=%s' % (self.table_name, self.table_id) + return u'' % \ + (self.group.name if self.group else repr(self.group), + table_info, self.capacity, self.state) class Group(vdm.sqlalchemy.RevisionedObjectMixin, vdm.sqlalchemy.StatefulObjectMixin, @@ -145,17 +156,55 @@ def set_approval_status(self, status): pass def get_children_groups(self, type='group'): - # Returns a list of dicts where each dict contains "id", "name", - # and "title" When querying with a CTE specifying a model in the - # query parameter causes problems as it returns only the first - # level deep apparently not recursing any deeper than that. If - # we simplify and request only specific fields then if returns - # the full depth of the hierarchy. - results = meta.Session.query("id", "name", "title").\ - from_statement(HIERARCHY_CTE).params(id=self.id, type=type).all() - return [{"id":idf, "name": name, "title": title} - for idf, name, title in results] + '''Returns the groups one level underneath this group in the hierarchy. + Groups come in a list of dicts, each keyed by "id", "name" and "title". + ''' + # The original intention of this method was to provide the full depth of + # the tree, but the CTE was incorrect. This new query does what that old CTE + # actually did, but is now far simpler. + results = meta.Session.query(Group.id, Group.name, Group.title).\ + filter_by(type=type).\ + join(Member, Member.table_id == Group.id).\ + filter_by(group=self).\ + filter_by(table_name='group').\ + filter_by(state='active').\ + all() + + return [{'id':id_, 'name': name, 'title': title} + for id_, name, title in results] + + def get_children_group_hierarchy(self, type='group'): + '''Returns the groups in all levels underneath this group in the hierarchy. + + :rtype: a list of tuples, each one a Group and the ID its their parent + group. + + e.g. >>> dept-health.get_children_group_hierarchy() + [(, u'8a163ba7-5146-4325-90c8-fe53b25e28d0'), + (, u'06e6dbf5-d801-40a1-9dc0-6785340b2ab4'), + (, u'd2e25b41-720c-4ba7-bc8f-bb34b185b3dd')] + ''' + results = meta.Session.query(Group, 'parent_id').\ + from_statement(HIERARCHY_DOWNWARDS_CTE).params(id=self.id, type=type).all() + return results + def get_parent_group_hierarchy(self, type='group'): + '''Returns this group's parent, parent's parent, parent's parent's parent + etc.. Sorted with the top level parent first.''' + return meta.Session.query(Group).\ + from_statement(HIERARCHY_UPWARDS_CTE).params(id=self.id, type=type).all() + + @classmethod + def get_top_level_groups(cls, type='group'): + '''Returns a list of the groups (of the specified type) which have + no parent groups.''' + return meta.Session.query(cls).\ + outerjoin(Member, Member.table_id == Group.id and \ + Member.table_name == 'group' and \ + Member.state == 'active').\ + filter(Member.id==None).\ + filter(Group.type==type).\ + order_by(Group.title).all() def packages(self, with_private=False, limit=None, return_query=False, context=None): @@ -248,6 +297,8 @@ def add_package_by_name(self, package_name): def get_groups(self, group_type=None, capacity=None): """ Get all groups that this group is within """ import ckan.model as model + # DR: Why is this cached? Surely the members can change in the + # lifetime of this Group? if '_groups' not in self.__dict__: self._groups = meta.Session.query(model.Group).\ join(model.Member, model.Member.group_id == model.Group.id and @@ -314,15 +365,33 @@ def __repr__(self): MemberRevision.related_packages = lambda self: [self.continuity.package] -HIERARCHY_CTE = """WITH RECURSIVE subtree(id) AS ( - SELECT M.* FROM public.member AS M - WHERE M.table_name = 'group' AND M.state = 'active' - UNION - SELECT M.* FROM public.member M, subtree SG - WHERE M.table_id = SG.group_id AND M.table_name = 'group' - AND M.state = 'active') - SELECT G.* FROM subtree AS ST - INNER JOIN public.group G ON G.id = ST.table_id - WHERE group_id = :id AND G.type = :type and table_name='group' - and G.state='active'""" +HIERARCHY_DOWNWARDS_CTE = """WITH RECURSIVE child AS +( + -- non-recursive term + SELECT * FROM member + WHERE group_id = :id AND table_name = 'group' AND state = 'active' + UNION ALL + -- recursive term + SELECT m.* FROM member AS m, child AS c + WHERE m.group_id = c.table_id AND m.table_name = 'group' + AND m.state = 'active' +) +SELECT G.*, child.group_id as parent_id FROM child + INNER JOIN public.group G ON G.id = child.table_id + WHERE G.type = :type AND G.state='active';""" + +HIERARCHY_UPWARDS_CTE = """WITH RECURSIVE parenttree(id) AS ( + -- non-recursive term + SELECT M.* FROM public.member AS M + WHERE table_id = :id AND M.table_name = 'group' AND M.state = 'active' + UNION + -- recursive term + SELECT M.* FROM public.member M + JOIN parenttree as PG ON PG.group_id = M.table_id + WHERE M.table_name = 'group' AND M.state = 'active' + ) + +SELECT G.* FROM parenttree AS PT + INNER JOIN public.group G ON G.id = PT.group_id + WHERE G.type = :type AND G.state='active';""" diff --git a/ckan/tests/models/test_group.py b/ckan/tests/models/test_group.py index 88d1e63d9be..000c5c41450 100644 --- a/ckan/tests/models/test_group.py +++ b/ckan/tests/models/test_group.py @@ -1,4 +1,4 @@ -from nose.tools import assert_equal +from ckan.tests import assert_equal, assert_not_in, assert_in import ckan.model as model from ckan.tests import * @@ -92,6 +92,68 @@ def _search_results(self, query, is_org=False): results = model.Group.search_by_name_or_title(query,is_org=is_org) return set([group.name for group in results]) +name_set_from_dicts = lambda groups: set([group['name'] for group in groups]) +name_set_from_group_tuple = lambda tuples: set([t[0].name for t in tuples]) +name_set_from_groups = lambda groups: set([group.name for group in groups]) +names_from_groups = lambda groups: [group.name for group in groups] + +class TestHierarchy: + @classmethod + def setup_class(self): + CreateTestData.create_group_hierarchy_test_data() + + def test_get_children_groups(self): + res = model.Group.by_name(u'department-of-health').\ + get_children_groups() + # check groups + assert_equal(name_set_from_dicts(res), + set(('national-health-service', + 'food-standards-agency'))) + # check each group is expressed as a small dict + assert_equal(set(res[0].keys()), set(('id', 'name', 'title'))) + assert_in(res[0]['name'], ('national-health-service', 'food-standards-agency')) + assert_in(res[0]['title'], ('National Health Service', 'Food Standards Agency')) + + def test_get_children_group_hierarchy__from_top(self): + assert_equal(name_set_from_group_tuple(model.Group.by_name(u'department-of-health').\ + get_children_group_hierarchy()), + set(('national-health-service', 'food-standards-agency', + 'nhs-wirral-ccg', 'nhs-southwark-ccg'))) + # i.e. not cabinet-office + + def test_get_children_group_hierarchy__from_tier_two(self): + assert_equal(name_set_from_group_tuple(model.Group.by_name(u'national-health-service').\ + get_children_group_hierarchy()), + set(('nhs-wirral-ccg', + 'nhs-southwark-ccg'))) + # i.e. not department-of-health or food-standards-agency + + def test_get_children_group_hierarchy__from_bottom_tier(self): + assert_equal(name_set_from_group_tuple(model.Group.by_name(u'nhs-wirral-ccg').\ + get_children_group_hierarchy()), + set()) + + def test_get_parent_groups_up_hierarchy__from_top(self): + assert_equal(names_from_groups(model.Group.by_name(u'department-of-health').\ + get_parent_group_hierarchy()), + []) + + def test_get_parent_groups_up_hierarchy__from_tier_two(self): + assert_equal(names_from_groups(model.Group.by_name(u'national-health-service').\ + get_parent_group_hierarchy()), + ['department-of-health']) + + def test_get_parent_groups_up_hierarchy__from_tier_three(self): + assert_equal(names_from_groups(model.Group.by_name(u'nhs-wirral-ccg').\ + get_parent_group_hierarchy()), + ['department-of-health', + 'national-health-service']) + + def test_get_top_level_groups(self): + assert_equal(names_from_groups(model.Group.by_name(u'nhs-wirral-ccg').\ + get_top_level_groups()), + ['cabinet-office', 'department-of-health']) + class TestGroupRevisions: @classmethod def setup_class(self): From 2c0bbafc509fc340dbdaa50c73e5ca712dc6b57a Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 23 Jul 2013 17:21:08 +0100 Subject: [PATCH 009/395] [#1038] Test data is now organizations rather than groups. --- ckan/lib/create_test_data.py | 34 +++++++++++++++++++++++++-------- ckan/tests/models/test_group.py | 18 +++++++++-------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index f464907d745..8b4d44707e1 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -231,8 +231,12 @@ def create_arbitrary(cls, package_dicts, relationships=[], # session has not yet been committed at this point. # Fetch from the new_groups dict instead. group = new_groups[group_name] - member = model.Member(group=group, table_id=pkg.id, table_name='package') + capacity = 'organization' if group.is_organization \ + else 'public' + member = model.Member(group=group, table_id=pkg.id, table_name='package', capacity=capacity) model.Session.add(member) + if group.is_organization: + pkg.owner_org = group.id elif attr == 'license': pkg.license_id = val elif attr == 'license_id': @@ -330,7 +334,8 @@ def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): else: admin_users = [] assert isinstance(group_dicts, (list, tuple)) - group_attributes = set(('name', 'title', 'description', 'parent_id')) + group_attributes = set(('name', 'title', 'description', 'parent_id', + 'type', 'is_organization')) for group_dict in group_dicts: if model.Group.by_name(unicode(group_dict['name'])): log.warning('Cannot create group "%s" as it already exists.' % \ @@ -861,26 +866,39 @@ def make_some_vocab_tags(cls): group_hierarchy_groups = [ {'name': 'department-of-health', 'title': 'Department of Health', - 'contact-email': 'contact@doh.gov.uk'}, + 'contact-email': 'contact@doh.gov.uk', + 'type': 'organization', + 'is_organization': True + }, {'name': 'food-standards-agency', 'title': 'Food Standards Agency', 'contact-email': 'contact@fsa.gov.uk', - 'parent': 'department-of-health'}, + 'parent': 'department-of-health', + 'type': 'organization', + 'is_organization': True}, {'name': 'national-health-service', 'title': 'National Health Service', 'contact-email': 'contact@nhs.gov.uk', - 'parent': 'department-of-health'}, + 'parent': 'department-of-health', + 'type': 'organization', + 'is_organization': True}, {'name': 'nhs-wirral-ccg', 'title': 'NHS Wirral CCG', 'contact-email': 'contact@wirral.nhs.gov.uk', - 'parent': 'national-health-service'}, + 'parent': 'national-health-service', + 'type': 'organization', + 'is_organization': True}, {'name': 'nhs-southwark-ccg', 'title': 'NHS Southwark CCG', 'contact-email': 'contact@southwark.nhs.gov.uk', - 'parent': 'national-health-service'}, + 'parent': 'national-health-service', + 'type': 'organization', + 'is_organization': True}, {'name': 'cabinet-office', 'title': 'Cabinet Office', - 'contact-email': 'contact@cabinet-office.gov.uk'}, + 'contact-email': 'contact@cabinet-office.gov.uk', + 'type': 'organization', + 'is_organization': True}, ] group_hierarchy_datasets = [ diff --git a/ckan/tests/models/test_group.py b/ckan/tests/models/test_group.py index 000c5c41450..13cee1907dc 100644 --- a/ckan/tests/models/test_group.py +++ b/ckan/tests/models/test_group.py @@ -97,6 +97,8 @@ def _search_results(self, query, is_org=False): name_set_from_groups = lambda groups: set([group.name for group in groups]) names_from_groups = lambda groups: [group.name for group in groups] +group_type = 'organization' + class TestHierarchy: @classmethod def setup_class(self): @@ -104,7 +106,7 @@ def setup_class(self): def test_get_children_groups(self): res = model.Group.by_name(u'department-of-health').\ - get_children_groups() + get_children_groups(type=group_type) # check groups assert_equal(name_set_from_dicts(res), set(('national-health-service', @@ -116,42 +118,42 @@ def test_get_children_groups(self): def test_get_children_group_hierarchy__from_top(self): assert_equal(name_set_from_group_tuple(model.Group.by_name(u'department-of-health').\ - get_children_group_hierarchy()), + get_children_group_hierarchy(type=group_type)), set(('national-health-service', 'food-standards-agency', 'nhs-wirral-ccg', 'nhs-southwark-ccg'))) # i.e. not cabinet-office def test_get_children_group_hierarchy__from_tier_two(self): assert_equal(name_set_from_group_tuple(model.Group.by_name(u'national-health-service').\ - get_children_group_hierarchy()), + get_children_group_hierarchy(type=group_type)), set(('nhs-wirral-ccg', 'nhs-southwark-ccg'))) # i.e. not department-of-health or food-standards-agency def test_get_children_group_hierarchy__from_bottom_tier(self): assert_equal(name_set_from_group_tuple(model.Group.by_name(u'nhs-wirral-ccg').\ - get_children_group_hierarchy()), + get_children_group_hierarchy(type=group_type)), set()) def test_get_parent_groups_up_hierarchy__from_top(self): assert_equal(names_from_groups(model.Group.by_name(u'department-of-health').\ - get_parent_group_hierarchy()), + get_parent_group_hierarchy(type=group_type)), []) def test_get_parent_groups_up_hierarchy__from_tier_two(self): assert_equal(names_from_groups(model.Group.by_name(u'national-health-service').\ - get_parent_group_hierarchy()), + get_parent_group_hierarchy(type=group_type)), ['department-of-health']) def test_get_parent_groups_up_hierarchy__from_tier_three(self): assert_equal(names_from_groups(model.Group.by_name(u'nhs-wirral-ccg').\ - get_parent_group_hierarchy()), + get_parent_group_hierarchy(type=group_type)), ['department-of-health', 'national-health-service']) def test_get_top_level_groups(self): assert_equal(names_from_groups(model.Group.by_name(u'nhs-wirral-ccg').\ - get_top_level_groups()), + get_top_level_groups(type=group_type)), ['cabinet-office', 'department-of-health']) class TestGroupRevisions: From 1229ff24d506a57f56ca4d030ab9228ff1d647c9 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 23 Jul 2013 17:22:27 +0100 Subject: [PATCH 010/395] [#1038] Docs for Member and its capacities is most handy. Explained the sort ordering for some of the group functions. --- ckan/model/group.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/ckan/model/group.py b/ckan/model/group.py index 2cf748bb078..a7de2495233 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -54,6 +54,19 @@ class Member(vdm.sqlalchemy.RevisionedObjectMixin, vdm.sqlalchemy.StatefulObjectMixin, domain_object.DomainObject): + '''A Member object represents any other object being a 'member' of a + particular Group. + + Meanings: + * Package - the Group is a collection of Packages + - capacity is 'public', 'private' + or 'organization' if the Group is an Organization + (see ckan.logic.action.package_owner_org_update) + * User - the User is granted permissions for the Group + - capacity is 'admin', 'editor' or 'member' + * Group - the Group (Member.group_id) is a child of the Group (Member.id) + in a hierarchy. + ''' def __init__(self, group=None, table_id=None, group_id=None, table_name=None, capacity='public', state='active'): self.group = group @@ -175,6 +188,7 @@ def get_children_groups(self, type='group'): def get_children_group_hierarchy(self, type='group'): '''Returns the groups in all levels underneath this group in the hierarchy. + The ordering is such that children always come after their parent. :rtype: a list of tuples, each one a Group and the ID its their parent group. @@ -197,7 +211,8 @@ def get_parent_group_hierarchy(self, type='group'): @classmethod def get_top_level_groups(cls, type='group'): '''Returns a list of the groups (of the specified type) which have - no parent groups.''' + no parent groups. Groups are sorted by title. + ''' return meta.Session.query(cls).\ outerjoin(Member, Member.table_id == Group.id and \ Member.table_name == 'group' and \ From 68bfd7aefc272bcd94cff226ce97067ec44a91a4 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 30 Jul 2013 16:18:07 +0100 Subject: [PATCH 011/395] [#1038] Permission cascading code with tests. --- ckan/lib/create_test_data.py | 24 ++-- ckan/logic/auth/update.py | 2 +- ckan/logic/validators.py | 3 +- ckan/new_authz.py | 34 ++++- ckan/tests/logic/test_auth.py | 235 ++++++++++++++++++++++++++++++++-- 5 files changed, 275 insertions(+), 23 deletions(-) diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 8b4d44707e1..80f32a1038d 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -40,9 +40,9 @@ def create_family_test_data(cls, extra_users=[]): @classmethod def create_group_hierarchy_test_data(cls, extra_users=[]): + cls.create_users(group_hierarchy_users) cls.create_groups(group_hierarchy_groups) - cls.create_arbitrary(group_hierarchy_datasets, - extra_user_names=group_hierarchy_users) + cls.create_arbitrary(group_hierarchy_datasets) @classmethod def create_test_user(cls): @@ -347,7 +347,7 @@ def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): for key in group_dict: if key in group_attributes: setattr(group, key, group_dict[key]) - else: + elif key not in ('admins', 'editors', 'parent'): group.extras[key] = group_dict[key] assert isinstance(pkg_names, (list, tuple)) for pkg_name in pkg_names: @@ -360,13 +360,13 @@ def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): for user_name in group_dict.get('admins', [])] \ + admin_users for admin in admins: - member = model.Member(group=group, table_id=user.id, + member = model.Member(group=group, table_id=admin.id, table_name='user', capacity='admin') model.Session.add(member) editors = [model.User.by_name(user_name) \ for user_name in group_dict.get('editors', [])] for editor in editors: - member = model.Member(group=group, table_id=user.id, + member = model.Member(group=group, table_id=editor.id, table_name='user', capacity='editor') model.Session.add(member) # Need to commit the current Group for two reasons: @@ -881,13 +881,17 @@ def make_some_vocab_tags(cls): 'contact-email': 'contact@nhs.gov.uk', 'parent': 'department-of-health', 'type': 'organization', - 'is_organization': True}, + 'is_organization': True, + 'editors': ['nhseditor'], + 'admins': ['nhsadmin']}, {'name': 'nhs-wirral-ccg', 'title': 'NHS Wirral CCG', 'contact-email': 'contact@wirral.nhs.gov.uk', 'parent': 'national-health-service', 'type': 'organization', - 'is_organization': True}, + 'is_organization': True, + 'editors': ['wirraleditor'], + 'admins': ['wirraladmin']}, {'name': 'nhs-southwark-ccg', 'title': 'NHS Southwark CCG', 'contact-email': 'contact@southwark.nhs.gov.uk', @@ -912,7 +916,11 @@ def make_some_vocab_tags(cls): 'groups': ['nhs-southwark-ccg']}, ] -group_hierarchy_users = ['nhsadmin', 'nhseditor', 'wirraladmin', 'wirraleditor'] +group_hierarchy_users = [{'name': 'nhsadmin', 'password': 'pass'}, + {'name': 'nhseditor', 'password': 'pass'}, + {'name': 'wirraladmin', 'password': 'pass'}, + {'name': 'wirraleditor', 'password': 'pass'}, + ] # Some test terms and translations. terms = ('A Novel By Tolstoy', diff --git a/ckan/logic/auth/update.py b/ckan/logic/auth/update.py index ac907f487d0..7376f27afb9 100644 --- a/ckan/logic/auth/update.py +++ b/ckan/logic/auth/update.py @@ -17,7 +17,7 @@ def package_update(context, data_dict): if package.owner_org: # if there is an owner org then we must have update_dataset - # premission for that organization + # permission for that organization check1 = new_authz.has_user_permission_for_group_or_org( package.owner_org, user, 'update_dataset' ) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index f9de157795c..0fb3887798e 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -36,7 +36,8 @@ def owner_org_validator(key, data, errors, context): group_id = group.id user = context['user'] user = model.User.get(user) - if not(user.sysadmin or user.is_in_group(group_id)): + if not(user.sysadmin or new_authz.has_user_permission_for_group_or_org( + group_id, user.name, 'create_dataset')): raise Invalid(_('You cannot add a dataset to this organization')) data[key] = group_id diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 1d391c50900..3296e795cf8 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -128,10 +128,14 @@ def get_roles_with_permission(permission): def has_user_permission_for_group_or_org(group_id, user_name, permission): - ''' Check if the user has the given permission for the group ''' + ''' Check if the user has the given permissions for the group, + allowing for sysadmin rights and permission cascading down groups. ''' if not group_id: return False - group_id = model.Group.get(group_id).id + group = model.Group.get(group_id) + if not group: + return False + group_id = group.id # Sys admins can do anything if is_sysadmin(user_name): @@ -140,12 +144,29 @@ def has_user_permission_for_group_or_org(group_id, user_name, permission): user_id = get_user_id_for_username(user_name, allow_none=True) if not user_id: return False + if _has_user_permission_for_groups(user_id, permission, [group_id]): + return True + # Handle when permissions cascade. Check the user's roles on groups higher + # in the group hierarchy for permission. + for capacity in check_config_permission('roles_that_cascade_to_sub_groups'): + parent_groups = group.get_parent_group_hierarchy(type=group.type) + group_ids = [group.id for group in parent_groups] + if _has_user_permission_for_groups(user_id, permission, group_ids, + capacity=capacity): + return True + return False + +def _has_user_permission_for_groups(user_id, permission, group_ids, capacity=None): + ''' Check if the user has the given permissions for the particular + group. Can also be filtered by a particular capacity''' # get any roles the user has for the group q = model.Session.query(model.Member) \ - .filter(model.Member.group_id == group_id) \ + .filter(model.Member.group_id.in_(group_ids)) \ .filter(model.Member.table_name == 'user') \ .filter(model.Member.state == 'active') \ .filter(model.Member.table_id == user_id) + if capacity: + q = q.filter(model.Member.capacity == capacity) # see if any role has the required permission # admin permission allows anything for the group for row in q.all(): @@ -281,18 +302,21 @@ def _get_auth_function(action): 'user_delete_groups': True, 'user_delete_organizations': True, 'create_user_via_api': False, + 'roles_that_cascade_to_sub_groups': ['admin'], } CONFIG_PERMISSIONS = {} def check_config_permission(permission): - ''' Returns the permission True/False based on config ''' + ''' Returns the permission configuration, usually True/False ''' # set up perms if not already done if not CONFIG_PERMISSIONS: for perm in CONFIG_PERMISSIONS_DEFAULTS: key = 'ckan.auth.' + perm default = CONFIG_PERMISSIONS_DEFAULTS[perm] - CONFIG_PERMISSIONS[perm] = asbool(config.get(key, default)) + CONFIG_PERMISSIONS[perm] = config.get(key, default) + if isinstance(perm, bool): + CONFIG_PERMISSIONS[perm] = asbool(CONFIG_PERMISSIONS[perm]) if permission in CONFIG_PERMISSIONS: return CONFIG_PERMISSIONS[permission] return False diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index 3ea9f2db39f..fcac3c1707b 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -1,7 +1,10 @@ +import paste.fixture + import ckan.tests as tests from ckan.logic import get_action import ckan.model as model import ckan.new_authz as new_authz +from ckan.lib.create_test_data import CreateTestData, group_hierarchy_groups import json INITIAL_TEST_CONFIG_PERMISSIONS = { @@ -13,6 +16,7 @@ 'user_delete_organizations': False, 'create_user_via_api': False, 'create_unowned_dataset': False, + 'roles_that_cascade_to_sub_groups': ['admin'], } @@ -33,19 +37,26 @@ def teardown_class(cls): new_authz.CONFIG_PERMISSIONS.update(cls.old_perm) model.repo.rebuild_db() - def _call_api(self, action, data, user, status=None): + @classmethod + def _call_api(cls, action, data, user, status=None): params = '%s=1' % json.dumps(data) - return self.app.post('/api/action/%s' % action, - params=params, - extra_environ={'Authorization': self.apikeys[user]}, - status=status) + res = cls.app.post('/api/action/%s' % action, + params=params, + extra_environ={'Authorization': cls.apikeys[user]}, + status=[200, 403, 409]) + if res.status != (status or 200): + error = json.loads(res.body)['error'] + raise AssertionError('Status was %s but should be %s. Error: %s' % + (res.status, status, error)) + return res - def create_user(self, name): + @classmethod + def create_user(cls, name): user = {'name': name, 'password': 'pass', 'email': 'moo@moo.com'} - res = self._call_api('user_create', user, 'sysadmin', 200) - self.apikeys[name] = str(json.loads(res.body)['result']['apikey']) + res = cls._call_api('user_create', user, 'sysadmin', 200) + cls.apikeys[name] = str(json.loads(res.body)['result']['apikey']) class TestAuthOrgs(TestAuth): @@ -190,6 +201,214 @@ def test_11_delete_org(self): self._call_api('organization_delete', org, 'org_editor', 403) self._call_api('organization_delete', org, 'org_admin', 403) +ORG_HIERARCHY_PERMISSIONS = { + 'roles_that_cascade_to_sub_groups': ['admin'], + } + +class TestAuthOrgHierarchy(TestAuth): + # Tests are in the same vein as TestAuthOrgs, testing the cases where the + # group hierarchy provices extra permissions through cascading + + @classmethod + def setup_class(cls): + TestAuth.setup_class() + CreateTestData.create_group_hierarchy_test_data() + for user in model.Session.query(model.User): + cls.apikeys[user.name] = str(user.apikey) + new_authz.CONFIG_PERMISSIONS.update(ORG_HIERARCHY_PERMISSIONS) + CreateTestData.create_arbitrary( + package_dicts= [{'name': 'adataset', + 'groups': ['national-health-service']}], + extra_user_names=['john']) + + def _reset_adatasets_owner_org(self): + rev = model.repo.new_revision() + get_action('package_owner_org_update')( + {'model': model, 'ignore_auth': True}, + {'id': 'adataset', + 'organization_id': 'national-health-service'}) + + def _undelete_package_if_needed(self, package_name): + pkg = model.Package.by_name(package_name) + if pkg and pkg.state == 'deleted': + rev = model.repo.new_revision() + pkg.state = 'active' + model.repo.commit_and_remove() + + def test_05_add_users_to_org_1(self): + member = {'username': 'john', 'role': 'admin', + 'id': 'department-of-health'} + self._call_api('organization_member_create', member, 'nhsadmin', 403) + def test_05_add_users_to_org_2(self): + member = {'username': 'john', 'role': 'editor', + 'id': 'department-of-health'} + self._call_api('organization_member_create', member, 'nhsadmin', 403) + def test_05_add_users_to_org_3(self): + member = {'username': 'john', 'role': 'admin', + 'id': 'national-health-service'} + self._call_api('organization_member_create', member, 'nhsadmin', 200) + def test_05_add_users_to_org_4(self): + member = {'username': 'john', 'role': 'editor', + 'id': 'national-health-service'} + self._call_api('organization_member_create', member, 'nhsadmin', 200) + def test_05_add_users_to_org_5(self): + member = {'username': 'john', 'role': 'admin', + 'id': 'nhs-wirral-ccg'} + self._call_api('organization_member_create', member, 'nhsadmin', 200) + def test_05_add_users_to_org_6(self): + member = {'username': 'john', 'role': 'editor', + 'id': 'nhs-wirral-ccg'} + self._call_api('organization_member_create', member, 'nhsadmin', 200) + def test_05_add_users_to_org_7(self): + member = {'username': 'john', 'role': 'editor', + 'id': 'national-health-service'} + self._call_api('organization_member_create', member, 'nhseditor', 403) + + def test_07_add_datasets_1(self): + dataset = {'name': 't1', 'owner_org': 'department-of-health'} + self._call_api('package_create', dataset, 'nhsadmin', 403) + + def test_07_add_datasets_2(self): + dataset = {'name': 't2', 'owner_org': 'national-health-service'} + self._call_api('package_create', dataset, 'nhsadmin', 200) + + def test_07_add_datasets_3(self): + dataset = {'name': 't3', 'owner_org': 'nhs-wirral-ccg'} + self._call_api('package_create', dataset, 'nhsadmin', 200) + + def test_07_add_datasets_4(self): + dataset = {'name': 't4', 'owner_org': 'department-of-health'} + self._call_api('package_create', dataset, 'nhseditor', 403) + + def test_07_add_datasets_5(self): + dataset = {'name': 't5', 'owner_org': 'national-health-service'} + self._call_api('package_create', dataset, 'nhseditor', 200) + + def test_07_add_datasets_6(self): + dataset = {'name': 't6', 'owner_org': 'nhs-wirral-ccg'} + self._call_api('package_create', dataset, 'nhseditor', 403) + + def test_08_update_datasets_1(self): + dataset = {'name': 'adataset', 'owner_org': 'department-of-health'} + self._call_api('package_update', dataset, 'nhsadmin', 409) + + def test_08_update_datasets_2(self): + dataset = {'name': 'adataset', 'owner_org': 'national-health-service'} + self._call_api('package_update', dataset, 'nhsadmin', 200) + + def test_08_update_datasets_3(self): + dataset = {'name': 'adataset', 'owner_org': 'nhs-wirral-ccg'} + try: + self._call_api('package_update', dataset, 'nhsadmin', 200) + finally: + self._reset_adatasets_owner_org() + + def test_08_update_datasets_4(self): + dataset = {'name': 'adataset', 'owner_org': 'department-of-health'} + self._call_api('package_update', dataset, 'nhseditor', 409) + + def test_08_update_datasets_5(self): + dataset = {'name': 'adataset', 'owner_org': 'national-health-service'} + try: + self._call_api('package_update', dataset, 'nhseditor', 200) + finally: + self._reset_adatasets_owner_org() + + def test_08_update_datasets_6(self): + dataset = {'name': 'adataset', 'owner_org': 'nhs-wirral-ccg'} + self._call_api('package_update', dataset, 'nhseditor', 409) + + def test_09_delete_datasets_1(self): + dataset = {'id': 'doh-spend'} + try: + self._call_api('package_delete', dataset, 'nhsadmin', 403) + finally: + self._undelete_package_if_needed(dataset['id']) + + def test_09_delete_datasets_2(self): + dataset = {'id': 'nhs-spend'} + try: + self._call_api('package_delete', dataset, 'nhsadmin', 200) + finally: + self._undelete_package_if_needed(dataset['id']) + + def test_09_delete_datasets_3(self): + dataset = {'id': 'wirral-spend'} + try: + self._call_api('package_delete', dataset, 'nhsadmin', 200) + finally: + self._undelete_package_if_needed(dataset['id']) + + def test_09_delete_datasets_4(self): + dataset = {'id': 'nhs-spend'} + try: + self._call_api('package_delete', dataset, 'nhseditor', 200) + finally: + self._undelete_package_if_needed(dataset['id']) + + def test_09_delete_datasets_5(self): + dataset = {'id': 'wirral-spend'} + try: + self._call_api('package_delete', dataset, 'nhseditor', 403) + finally: + self._undelete_package_if_needed(dataset['id']) + + def _flesh_out_organization(self, org): + # When calling organization_update, unless you include the list of + # editor and admin users and parent groups, it will remove them. So + # get the current list + existing_org = get_action('organization_show')( + {'model': model, 'ignore_auth': True}, {'id': org['id']}) + org.update(existing_org) + + def test_10_edit_org_1(self): + org = {'id': 'department-of-health', 'title': 'test'} + self._flesh_out_organization(org) + self._call_api('organization_update', org, 'nhsadmin', 403) + + def test_10_edit_org_2(self): + org = {'id': 'national-health-service', 'title': 'test'} + self._flesh_out_organization(org) + import pprint; pprint.pprint(org) + print model.Session.query(model.Member).filter_by(state='deleted').all() + self._call_api('organization_update', org, 'nhsadmin', 200) + print model.Session.query(model.Member).filter_by(state='deleted').all() + + def test_10_edit_org_3(self): + org = {'id': 'nhs-wirral-ccg', 'title': 'test'} + self._flesh_out_organization(org) + self._call_api('organization_update', org, 'nhsadmin', 200) + + def test_10_edit_org_4(self): + org = {'id': 'department-of-health', 'title': 'test'} + self._flesh_out_organization(org) + self._call_api('organization_update', org, 'nhseditor', 403) + + def test_10_edit_org_5(self): + org = {'id': 'national-health-service', 'title': 'test'} + self._flesh_out_organization(org) + self._call_api('organization_update', org, 'nhseditor', 403) + + def test_10_edit_org_6(self): + org = {'id': 'nhs-wirral-ccg', 'title': 'test'} + self._flesh_out_organization(org) + self._call_api('organization_update', org, 'nhseditor', 403) + + def test_11_delete_org_1(self): + org = {'id': 'department-of-health'} + self._call_api('organization_delete', org, 'nhsadmin', 403) + self._call_api('organization_delete', org, 'nhseditor', 403) + + def test_11_delete_org_2(self): + org = {'id': 'national-health-service'} + self._call_api('organization_delete', org, 'nhsadmin', 403) + self._call_api('organization_delete', org, 'nhseditor', 403) + + def test_11_delete_org_3(self): + org = {'id': 'nhs-wirral-ccg'} + self._call_api('organization_delete', org, 'nhsadmin', 403) + self._call_api('organization_delete', org, 'nhseditor', 403) + class TestAuthGroups(TestAuth): From 4db2d24908fc382a7548fbbcc7dc5133dcdd29ad Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 30 Jul 2013 16:44:59 +0100 Subject: [PATCH 012/395] [#1038] Fix permission checking for organizations. Corrected bad test. --- ckan/logic/auth/create.py | 2 +- ckan/tests/logic/test_auth.py | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index bf9c3d17ea3..299308a9ae5 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -23,7 +23,7 @@ def package_create(context, data_dict=None): # If an organization is given are we able to add a dataset to it? data_dict = data_dict or {} - org_id = data_dict.get('organization_id') + org_id = data_dict.get('owner_org') if org_id and not new_authz.has_user_permission_for_group_or_org( org_id, user, 'create_dataset'): return {'success': False, 'msg': _('User %s not authorized to add dataset to this organization') % user} diff --git a/ckan/tests/logic/test_auth.py b/ckan/tests/logic/test_auth.py index fcac3c1707b..2dc6b5e9a5a 100644 --- a/ckan/tests/logic/test_auth.py +++ b/ckan/tests/logic/test_auth.py @@ -60,6 +60,8 @@ def create_user(cls, name): class TestAuthOrgs(TestAuth): + # NB: These tests are dependent on each other, so don't run them + # separately. def test_01_create_users(self): # actual roles assigned later @@ -90,6 +92,7 @@ def test_02_create_orgs(self): def test_03_create_dataset_no_org(self): + # no owner_org supplied dataset = {'name': 'admin_create_no_org'} self._call_api('package_create', dataset, 'sysadmin', 409) @@ -106,7 +109,7 @@ def test_04_create_dataset_with_org(self): 'owner_org': 'org_no_user'} self._call_api('package_create', dataset, 'sysadmin', 200) - dataset = {'name': 'user_create_with_org', + dataset = {'name': 'user_create_with_no_org', 'owner_org': 'org_with_user'} self._call_api('package_create', dataset, 'no_org', 403) @@ -138,7 +141,7 @@ def _add_datasets(self, user): #not able to add dataset to org admin does not belong to. dataset = {'name': user + '_dataset_bad', 'owner_org': 'org_no_user'} - self._call_api('package_create', dataset, user, 409) + self._call_api('package_create', dataset, user, 403) #admin not able to make dataset not owned by a org dataset = {'name': user + '_dataset_bad'} @@ -146,7 +149,7 @@ def _add_datasets(self, user): #not able to add org to not existant org dataset = {'name': user + '_dataset_bad', 'owner_org': 'org_not_exist'} - self._call_api('package_create', dataset, user, 409) + self._call_api('package_create', dataset, user, 403) def test_07_add_datasets(self): self._add_datasets('org_admin') @@ -317,7 +320,7 @@ def test_08_update_datasets_5(self): def test_08_update_datasets_6(self): dataset = {'name': 'adataset', 'owner_org': 'nhs-wirral-ccg'} self._call_api('package_update', dataset, 'nhseditor', 409) - + def test_09_delete_datasets_1(self): dataset = {'id': 'doh-spend'} try: From 6d97199ca314f5130f5318d23fc6a2f04e6d7ab5 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 30 Jul 2013 16:45:52 +0100 Subject: [PATCH 013/395] [#1038] Fix unreliable ordering of upward CTE. --- ckan/model/group.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ckan/model/group.py b/ckan/model/group.py index a7de2495233..f21bafb3cb8 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -396,17 +396,17 @@ def __repr__(self): INNER JOIN public.group G ON G.id = child.table_id WHERE G.type = :type AND G.state='active';""" -HIERARCHY_UPWARDS_CTE = """WITH RECURSIVE parenttree(id) AS ( +HIERARCHY_UPWARDS_CTE = """WITH RECURSIVE parenttree(depth) AS ( -- non-recursive term - SELECT M.* FROM public.member AS M + SELECT 0, M.* FROM public.member AS M WHERE table_id = :id AND M.table_name = 'group' AND M.state = 'active' UNION -- recursive term - SELECT M.* FROM public.member M - JOIN parenttree as PG ON PG.group_id = M.table_id - WHERE M.table_name = 'group' AND M.state = 'active' + SELECT PG.depth + 1, M.* FROM parenttree PG, public.member M + WHERE PG.group_id = M.table_id AND M.table_name = 'group' AND M.state = 'active' ) -SELECT G.* FROM parenttree AS PT +SELECT G.*, PT.depth FROM parenttree AS PT INNER JOIN public.group G ON G.id = PT.group_id - WHERE G.type = :type AND G.state='active';""" + WHERE G.type = :type AND G.state='active' + ORDER BY PT.depth DESC;""" From 1e7b07bb9624e975f88715a00388f84db74d4da3 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 15 Aug 2013 19:07:01 +0200 Subject: [PATCH 014/395] First work on theming docs and example --- ckanext/example_theme/__init__.py | 0 ckanext/example_theme/plugin.py | 9 ++ doc/theming.rst | 157 +++++++++++++++++++++++++++++- setup.py | 1 + 4 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 ckanext/example_theme/__init__.py create mode 100644 ckanext/example_theme/plugin.py diff --git a/ckanext/example_theme/__init__.py b/ckanext/example_theme/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/plugin.py b/ckanext/example_theme/plugin.py new file mode 100644 index 00000000000..94bcae49f33 --- /dev/null +++ b/ckanext/example_theme/plugin.py @@ -0,0 +1,9 @@ +import ckan.plugins as plugins + + +class ExampleThemePlugin(plugins.SingletonPlugin): + '''An example plugin that just registers the directories containing our + example theme files. + + ''' + pass diff --git a/doc/theming.rst b/doc/theming.rst index 5b539dbb238..d61a7484461 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -2,12 +2,159 @@ Theming ======= -If you want more control over your CKAN site's layout and appearance than the -options described in :doc:`getting-started` give, you can further customize -CKAN's appearance by developing a theme. CKAN's templates, HTML and CSS are all -completely customizable by themes. This document will walk you through the -process of developing a CKAN theme. +.. todo:: + Add more to :doc:`getting-started`, is there more that can be done in the + config file, e.g. site logo? + +:doc:`getting-started` documents some simple CKAN configuration settings that +you can use to, for example, change the title of your CKAN site. For those who +want more control over their CKAN site's frontend, this document covers +everything you need to know to develop a custom CKAN theme, including how to +customize CKAN's HTML templates, CSS and |javascript|. The sections below will +walk you through the process of creating a simple, example CKAN theme that +demonstrates all of the main features of CKAN theming. + +.. todo:: + + Insert link to the completed example theme here. + + +--------------- +Installing CKAN +--------------- + +Before you can start developing a CKAN theme, you’ll need a working source +install of CKAN on your system. If you don’t have a CKAN source install +already, follow the instructions in :doc:`install-from-source` before +continuing. + + +------------------------- +Creating a CKAN extension +------------------------- + +A CKAN theme must be contained within a CKAN extension, so we'll begin by +creating an extension to hold our theme. As documented in +:doc:`writing-extensions`, extensions can customize and extend CKAN's features +in many powerful ways, but in this example we'll use our extension only to hold +our theme. + +.. todo:: + + This stuff is duplicated from the writing extensions docs, do a Sphinx + include here instead. + +First, use the ``paster create`` command to create an empty extension: + +.. parsed-literal:: + + |activate| + cd |virtualenv|/src + paster --plugin=ckan create -t ckanext ckanext-example_theme + +The command will ask you to answer a few questions. The answers you give will +end up in your extension's ``setup.py`` file (where you can edit them later if +you want). + +(See :doc:`writing-extensions` for full documentation on creating CKAN +extensions and plugins.) + +Now create the file ``ckanext-example_theme/ckanext/example_theme/plugin.py`` +with the following contents: + +.. literalinclude:: ../ckanext/example_theme/plugin.py + +Now let's add our plugin to the ``entry_points`` in ``setup.py``. This +identifies the plugin to CKAN once the extension is installed in CKAN's +virtualenv. Edit ``ckanext-example_theme/setup.py`` and add a line to the +``entry_points`` section like this:: + + entry_points=''' + [ckan.plugins] + example_theme=ckanext.example_theme.plugin:ExampleThemePlugin + ''', + +Install the ``example_theme`` extension: + +.. parsed-literal:: + + |activate| + cd |virtualenv|/src/ckanext-example_theme + python setup.py develop + +Finally, enable the plugin in your CKAN config file. Edit |development.ini| and +add ``example_theme`` to the ``ckan.plugins`` line, for example:: + + ckan.plugins = stats text_preview recline_preview example_theme + +You should now be able to start CKAN in the development web server and have it +start up without any problems: + +.. parsed-literal:: + + $ paster serve |development.ini| + Starting server in PID 13961. + serving on 0.0.0.0:5000 view at http://127.0.0.1:5000 + +If your plugin is in the :ref:`ckan.plugins` setting and CKAN starts without +crashing, then your plugin is installed and CKAN can find it. Of course, your +plugin doesn't *do* anything yet. + + +-------------------------------------------- +Customizing CKAN's HTML and Jinja2 templates +-------------------------------------------- + +.. todo:: + + * Introduce Bootstrap here. A lot of Bootstrap stuff can be done just using + HTML. It should also get mentioned in other sections (CSS, JavaScript..) + * HTML (which version?) + * Jinja2 + * CKAN's custom Jinja2 tags and form macros + + +--------------------------------------------------------------------- +Adding CSS, JavaScript, images and other static files using Fanstatic +--------------------------------------------------------------------- + +.. todo:: + + * Introduce Fanstatic + * Use the plugin to register a Fanstatic library + * Use ``{% resource %}`` to load stuff from the library + * Presumably you can also load stuff from the core library? + + +---------------------- +Customizing CKAN's CSS +---------------------- + +.. todo:: + + * Introduce CSS? + * Use Fanstatic to add a CSS file + * Use Bootstrap's CSS files and CKAN core's + * See the CKAN style guide + + +----------------------------- +Customizing CKAN's JavaScript +----------------------------- + +.. todo:: + + * How to load JavaScript modules + * jQuery + * Bootstrap's JavaScript stuff + * Other stuff in javascript-module-tutorial.rst + + + + + +---- Create Custom Extension ----------------------- diff --git a/setup.py b/setup.py index 276e03a5fdf..8588da6a0e1 100644 --- a/setup.py +++ b/setup.py @@ -122,6 +122,7 @@ recline_preview=ckanext.reclinepreview.plugin:ReclinePreview example_itemplatehelpers=ckanext.example_itemplatehelpers.plugin:ExampleITemplateHelpersPlugin example_idatasetform=ckanext.example_idatasetform.plugin:ExampleIDatasetFormPlugin + example_theme=ckanext.example_theme.plugin:ExampleThemePlugin [ckan.system_plugins] domain_object_mods = ckan.model.modification:DomainObjectModificationExtension From 9a70373bdaa2626ba1a0f476fc1b6fc1f06c5764 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 21 Aug 2013 21:05:12 +0200 Subject: [PATCH 015/395] [#847] Add a TODO --- doc/theming.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/theming.rst b/doc/theming.rst index d61a7484461..ba477217a1e 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -137,6 +137,8 @@ Customizing CKAN's CSS * Use Fanstatic to add a CSS file * Use Bootstrap's CSS files and CKAN core's * See the CKAN style guide + * custom.less: Edit ckan/public/base/less/custom.less and then run ./bin/less + or ./bin/less --production ----------------------------- From 7b57f322c21b175e2f13035b68367171e65bdde7 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 3 Sep 2013 11:41:01 +0100 Subject: [PATCH 016/395] [#1218] Ignore __extras on package_show extras --- ckan/logic/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f8fe8df7fbd..44c8c3797bf 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -340,6 +340,7 @@ def default_extras_schema(): 'state': [ignore], 'deleted': [ignore_missing], 'revision_timestamp': [ignore], + '__extras': [ignore], } return schema From 1fce2cc1d499c3cb5421732b08c3ee518ca18be9 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 4 Sep 2013 17:19:39 +0200 Subject: [PATCH 017/395] [#847] More work on the new theming docs Completed first draft of most of the templates stuff. Still needs a lot of work. --- ckanext/example_theme/v1/__init__.py | 0 ckanext/example_theme/{ => v1}/plugin.py | 3 +- ckanext/example_theme/v2/__init__.py | 0 ckanext/example_theme/v2/plugin.py | 14 + .../v2/templates/home/index.html | 0 ckanext/example_theme/v3/__init__.py | 0 ckanext/example_theme/v3/plugin.py | 1 + .../v3/templates/home/index.html | 1 + ckanext/example_theme/v4/__init__.py | 0 ckanext/example_theme/v4/plugin.py | 1 + .../v4/templates/home/index.html | 5 + ckanext/example_theme/v6/__init__.py | 0 ckanext/example_theme/v6/plugin.py | 1 + .../v6/templates/home/index.html | 5 + ckanext/example_theme/v7/__init__.py | 0 ckanext/example_theme/v7/plugin.py | 34 ++ .../v7/templates/home/index.html | 11 + ckanext/example_theme/v8/__init__.py | 0 ckanext/example_theme/v8/plugin.py | 1 + .../v8/templates/home/index.html | 10 + ckanext/example_theme/v9/__init__.py | 0 ckanext/example_theme/v9/plugin.py | 1 + .../v9/templates/home/index.html | 10 + .../example_theme_dataset_of_the_day.html | 15 + doc/template-helper-functions.rst | 7 + doc/theming.rst | 521 +++++++++++++----- setup.py | 9 +- 27 files changed, 502 insertions(+), 148 deletions(-) create mode 100644 ckanext/example_theme/v1/__init__.py rename ckanext/example_theme/{ => v1}/plugin.py (50%) create mode 100644 ckanext/example_theme/v2/__init__.py create mode 100644 ckanext/example_theme/v2/plugin.py create mode 100644 ckanext/example_theme/v2/templates/home/index.html create mode 100644 ckanext/example_theme/v3/__init__.py create mode 120000 ckanext/example_theme/v3/plugin.py create mode 100644 ckanext/example_theme/v3/templates/home/index.html create mode 100644 ckanext/example_theme/v4/__init__.py create mode 120000 ckanext/example_theme/v4/plugin.py create mode 100644 ckanext/example_theme/v4/templates/home/index.html create mode 100644 ckanext/example_theme/v6/__init__.py create mode 120000 ckanext/example_theme/v6/plugin.py create mode 100644 ckanext/example_theme/v6/templates/home/index.html create mode 100644 ckanext/example_theme/v7/__init__.py create mode 100644 ckanext/example_theme/v7/plugin.py create mode 100644 ckanext/example_theme/v7/templates/home/index.html create mode 100644 ckanext/example_theme/v8/__init__.py create mode 120000 ckanext/example_theme/v8/plugin.py create mode 100644 ckanext/example_theme/v8/templates/home/index.html create mode 100644 ckanext/example_theme/v9/__init__.py create mode 120000 ckanext/example_theme/v9/plugin.py create mode 100644 ckanext/example_theme/v9/templates/home/index.html create mode 100644 ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html create mode 100644 doc/template-helper-functions.rst diff --git a/ckanext/example_theme/v1/__init__.py b/ckanext/example_theme/v1/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/plugin.py b/ckanext/example_theme/v1/plugin.py similarity index 50% rename from ckanext/example_theme/plugin.py rename to ckanext/example_theme/v1/plugin.py index 94bcae49f33..43686f7e315 100644 --- a/ckanext/example_theme/plugin.py +++ b/ckanext/example_theme/v1/plugin.py @@ -2,8 +2,7 @@ class ExampleThemePlugin(plugins.SingletonPlugin): - '''An example plugin that just registers the directories containing our - example theme files. + '''An example theme plugin. ''' pass diff --git a/ckanext/example_theme/v2/__init__.py b/ckanext/example_theme/v2/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v2/plugin.py b/ckanext/example_theme/v2/plugin.py new file mode 100644 index 00000000000..bf47f41095a --- /dev/null +++ b/ckanext/example_theme/v2/plugin.py @@ -0,0 +1,14 @@ +import ckan.plugins as plugins + + +class ExampleThemePlugin(plugins.SingletonPlugin): + '''An example theme plugin. + + ''' + plugins.implements(plugins.IConfigurer) + + def update_config(self, config): + + # Add this plugin's templates dir to CKAN's extra_template_paths, so + # that CKAN will use this plugin's custom templates. + toolkit.add_template_directory(config, 'templates') diff --git a/ckanext/example_theme/v2/templates/home/index.html b/ckanext/example_theme/v2/templates/home/index.html new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v3/__init__.py b/ckanext/example_theme/v3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v3/plugin.py b/ckanext/example_theme/v3/plugin.py new file mode 120000 index 00000000000..e2729b2750a --- /dev/null +++ b/ckanext/example_theme/v3/plugin.py @@ -0,0 +1 @@ +../v2/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v3/templates/home/index.html b/ckanext/example_theme/v3/templates/home/index.html new file mode 100644 index 00000000000..c58cded7bfa --- /dev/null +++ b/ckanext/example_theme/v3/templates/home/index.html @@ -0,0 +1 @@ +{% ckan_extends %} diff --git a/ckanext/example_theme/v4/__init__.py b/ckanext/example_theme/v4/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v4/plugin.py b/ckanext/example_theme/v4/plugin.py new file mode 120000 index 00000000000..e2729b2750a --- /dev/null +++ b/ckanext/example_theme/v4/plugin.py @@ -0,0 +1 @@ +../v2/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v4/templates/home/index.html b/ckanext/example_theme/v4/templates/home/index.html new file mode 100644 index 00000000000..7d52a4b6547 --- /dev/null +++ b/ckanext/example_theme/v4/templates/home/index.html @@ -0,0 +1,5 @@ +{% ckan_extends %} + +{% block secondary_content %} + Hello block world! +{% endblock %} diff --git a/ckanext/example_theme/v6/__init__.py b/ckanext/example_theme/v6/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v6/plugin.py b/ckanext/example_theme/v6/plugin.py new file mode 120000 index 00000000000..e2729b2750a --- /dev/null +++ b/ckanext/example_theme/v6/plugin.py @@ -0,0 +1 @@ +../v2/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v6/templates/home/index.html b/ckanext/example_theme/v6/templates/home/index.html new file mode 100644 index 00000000000..acf374b1d22 --- /dev/null +++ b/ckanext/example_theme/v6/templates/home/index.html @@ -0,0 +1,5 @@ +{% ckan_extends %} + +{% block secondary_content %} + {{ h.new_packages_activity_stream() }} +{% endblock %} diff --git a/ckanext/example_theme/v7/__init__.py b/ckanext/example_theme/v7/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v7/plugin.py b/ckanext/example_theme/v7/plugin.py new file mode 100644 index 00000000000..bfe2a15572a --- /dev/null +++ b/ckanext/example_theme/v7/plugin.py @@ -0,0 +1,34 @@ +import random + +import ckan.plugins as plugins +import ckan.plugins.toolkit as toolkit + + +def example_theme_dataset_of_the_day(): + '''Return the dataset of the day. + + ''' + dataset_names = toolkit.get_action('package_list')(data_dict={}) + dataset_name = random.choice(dataset_names) + dataset = toolkit.get_action('package_show')( + data_dict={'id': dataset_name}) + return dataset + + +class ExampleThemePlugin(plugins.SingletonPlugin): + '''An example theme plugin. + + ''' + plugins.implements(plugins.IConfigurer) + plugins.implements(plugins.ITemplateHelpers) + + def update_config(self, config): + + # Add this plugin's templates dir to CKAN's extra_template_paths, so + # that CKAN will use this plugin's custom templates. + toolkit.add_template_directory(config, 'templates') + + def get_helpers(self): + + return {'example_theme_dataset_of_the_day': + example_theme_dataset_of_the_day} diff --git a/ckanext/example_theme/v7/templates/home/index.html b/ckanext/example_theme/v7/templates/home/index.html new file mode 100644 index 00000000000..be81cfaebc7 --- /dev/null +++ b/ckanext/example_theme/v7/templates/home/index.html @@ -0,0 +1,11 @@ +{% ckan_extends %} + +{% block secondary_content %} + {{ h.recently_changed_packages_activity_stream() }} + +

+ Dataset of the day: + {{ h.example_theme_dataset_of_the_day().title }} +

+ +{% endblock %} diff --git a/ckanext/example_theme/v8/__init__.py b/ckanext/example_theme/v8/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v8/plugin.py b/ckanext/example_theme/v8/plugin.py new file mode 120000 index 00000000000..6f55fa44e01 --- /dev/null +++ b/ckanext/example_theme/v8/plugin.py @@ -0,0 +1 @@ +../v7/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v8/templates/home/index.html b/ckanext/example_theme/v8/templates/home/index.html new file mode 100644 index 00000000000..6cbf40da9a2 --- /dev/null +++ b/ckanext/example_theme/v8/templates/home/index.html @@ -0,0 +1,10 @@ +{% ckan_extends %} + +{% block secondary_content %} + + {{ h.recently_changed_packages_activity_stream() }} + + {% snippet 'snippets/package_item.html', + package=h.example_theme_dataset_of_the_day() %} + +{% endblock %} diff --git a/ckanext/example_theme/v9/__init__.py b/ckanext/example_theme/v9/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v9/plugin.py b/ckanext/example_theme/v9/plugin.py new file mode 120000 index 00000000000..74b99bdf60c --- /dev/null +++ b/ckanext/example_theme/v9/plugin.py @@ -0,0 +1 @@ +../v8/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v9/templates/home/index.html b/ckanext/example_theme/v9/templates/home/index.html new file mode 100644 index 00000000000..2cf4f00533d --- /dev/null +++ b/ckanext/example_theme/v9/templates/home/index.html @@ -0,0 +1,10 @@ +{% ckan_extends %} + +{% block secondary_content %} + + {{ h.recently_changed_packages_activity_stream() }} + + {% snippet 'snippets/example_theme_dataset_of_the_day.html', + dataset=h.example_theme_dataset_of_the_day() %} + +{% endblock %} diff --git a/ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html b/ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html new file mode 100644 index 00000000000..102a01e18b9 --- /dev/null +++ b/ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html @@ -0,0 +1,15 @@ +{# Renders a preview of the dataset of the day. + +dataset - The dataset to preview + +#} +{% set notes = h.markdown_extract(dataset.notes, extract_length=truncate) %} +
+

+ {{ h.link_to(h.truncate(title, truncate_title), + h.url_for(controller='package', action='read', id=dataset.name)) }} +

+ {% if notes %} +
{{ notes|urlize }}
+ {% endif %} +
diff --git a/doc/template-helper-functions.rst b/doc/template-helper-functions.rst new file mode 100644 index 00000000000..77a36f972dc --- /dev/null +++ b/doc/template-helper-functions.rst @@ -0,0 +1,7 @@ +========================= +Template helper functions +========================= + +.. automodule:: ckan.lib.helpers + :members: + :undoc-members: diff --git a/doc/theming.rst b/doc/theming.rst index ba477217a1e..c129fe31632 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -1,3 +1,5 @@ +.. _Jinja2: http://jinja.pocoo.org/ + ======= Theming ======= @@ -7,22 +9,25 @@ Theming Add more to :doc:`getting-started`, is there more that can be done in the config file, e.g. site logo? -:doc:`getting-started` documents some simple CKAN configuration settings that -you can use to, for example, change the title of your CKAN site. For those who -want more control over their CKAN site's frontend, this document covers -everything you need to know to develop a custom CKAN theme, including how to -customize CKAN's HTML templates, CSS and |javascript|. The sections below will -walk you through the process of creating a simple, example CKAN theme that -demonstrates all of the main features of CKAN theming. +The CKAN frontend can be fully customized by developing a CKAN theme. +If you just want to do some simple customizations such as changing the title +of your CKAN site, or making some small CSS customizations, +:doc:`getting-started` documents some simple configuration settings you can +use. +If you want more control, follow the tutorial below to learn how to develop +your custom CKAN theme. -.. todo:: - Insert link to the completed example theme here. +---------------- +Theming tutorial +---------------- + +This tutorial walks you through the process of creating a CKAN theme, and +demonstrates all of the main features of CKAN theming. ---------------- Installing CKAN ---------------- +=============== Before you can start developing a CKAN theme, you’ll need a working source install of CKAN on your system. If you don’t have a CKAN source install @@ -30,211 +35,437 @@ already, follow the instructions in :doc:`install-from-source` before continuing. -------------------------- Creating a CKAN extension -------------------------- +========================= -A CKAN theme must be contained within a CKAN extension, so we'll begin by -creating an extension to hold our theme. As documented in -:doc:`writing-extensions`, extensions can customize and extend CKAN's features -in many powerful ways, but in this example we'll use our extension only to hold -our theme. +A CKAN theme is simply a CKAN plugin that contains some custom templates and +static files, so before getting started on our CKAN theme we'll have to create +an extension and plugin. For a detailed explanation of the steps below, see +:doc:`writing-extensions`. -.. todo:: - - This stuff is duplicated from the writing extensions docs, do a Sphinx - include here instead. - -First, use the ``paster create`` command to create an empty extension: - -.. parsed-literal:: +1. Use the ``paster create`` command to create an empty extension: - |activate| - cd |virtualenv|/src - paster --plugin=ckan create -t ckanext ckanext-example_theme + .. parsed-literal:: -The command will ask you to answer a few questions. The answers you give will -end up in your extension's ``setup.py`` file (where you can edit them later if -you want). + |activate| + cd |virtualenv|/src + paster --plugin=ckan create -t ckanext ckanext-example_theme -(See :doc:`writing-extensions` for full documentation on creating CKAN -extensions and plugins.) +2. Create the file ``ckanext-example_theme/ckanext/example_theme/plugin.py`` + with the following contents: -Now create the file ``ckanext-example_theme/ckanext/example_theme/plugin.py`` -with the following contents: + .. literalinclude:: ../ckanext/example_theme/v1/plugin.py -.. literalinclude:: ../ckanext/example_theme/plugin.py - -Now let's add our plugin to the ``entry_points`` in ``setup.py``. This -identifies the plugin to CKAN once the extension is installed in CKAN's -virtualenv. Edit ``ckanext-example_theme/setup.py`` and add a line to the -``entry_points`` section like this:: +3. Edit the ``entry_points`` in ``ckanext-example_theme/setup.py``:: entry_points=''' [ckan.plugins] example_theme=ckanext.example_theme.plugin:ExampleThemePlugin ''', -Install the ``example_theme`` extension: +4. Run ``python setup.py develop``: -.. parsed-literal:: + .. parsed-literal:: - |activate| - cd |virtualenv|/src/ckanext-example_theme - python setup.py develop + |activate| + cd |virtualenv|/src/ckanext-example_theme + python setup.py develop -Finally, enable the plugin in your CKAN config file. Edit |development.ini| and -add ``example_theme`` to the ``ckan.plugins`` line, for example:: +5. Add the plugin to the ``ckan.plugins`` setting in your |development.ini| + file:: ckan.plugins = stats text_preview recline_preview example_theme -You should now be able to start CKAN in the development web server and have it -start up without any problems: +6. Start CKAN in the development web server: -.. parsed-literal:: + .. parsed-literal:: $ paster serve |development.ini| Starting server in PID 13961. serving on 0.0.0.0:5000 view at http://127.0.0.1:5000 -If your plugin is in the :ref:`ckan.plugins` setting and CKAN starts without -crashing, then your plugin is installed and CKAN can find it. Of course, your -plugin doesn't *do* anything yet. + If your plugin is in the :ref:`ckan.plugins` setting and CKAN starts without + crashing, then your plugin is installed and CKAN can find it. Of course, + your plugin doesn't *do* anything yet. --------------------------------------------- -Customizing CKAN's HTML and Jinja2 templates --------------------------------------------- +Customizing CKAN's HTML with Jinja2 +=================================== -.. todo:: - * Introduce Bootstrap here. A lot of Bootstrap stuff can be done just using - HTML. It should also get mentioned in other sections (CSS, JavaScript..) - * HTML (which version?) - * Jinja2 - * CKAN's custom Jinja2 tags and form macros +Replacing a default template file +--------------------------------- +Every CKAN page is generated by rendering a particular template. For each +page of a CKAN site there's a corresponding template file. For example the +front page is generated from the ``ckan/templates/home/index.html`` file, the +``/about`` page is generated from ``ckan/templates/home/about.html``, the +datasets page at ``/dataset`` is generated from +``ckan/templates/package/search.html``, etc. ---------------------------------------------------------------------- -Adding CSS, JavaScript, images and other static files using Fanstatic ---------------------------------------------------------------------- +.. todo:: + + Explain how to find out which template file is used for a given page. + +To customize pages, our plugin needs to register its own custom template +directory containing templates file that override the default ones. +Edit the ``plugin.py`` file that we created earlier, so that it looks like +this: + +.. literalinclude:: ../ckanext/example_theme/v2/plugin.py + +This new code does a few things: + +1. It imports CKAN's *plugins toolkit* module: + + .. literalinclude:: ../ckanext/example_theme/plugin_v2.py + :start-after: import ckan.plugins as plugins + :end-before: class ExampleThemePlugin(plugins.SingletonPlugin): + + The plugins toolkit is a Python module containing core functions, classes + and exceptions for CKAN plugins to use. For more about the plugins toolkit, + see :doc:`writing-extensions`. + +2. It calls :py:func:`~ckan.plugins.core.implements` to declare that it + implements the :py:class:`~ckan.plugins.interfaces.IConfigurer` plugin + interface. This tells CKAN that our + :py:class:`~ckanext.example_theme.plugin_v2.ExampleThemePlugin` class + implements the methods declared in the + :py:class:`~ckan.plugins.interfaces.IConfigurer` interface. CKAN will call + these methods of our plugin class at the appropriate times. + +3. It implements the + :py:meth:`~ckan.plugins.interfaces.IConfigurer.update_config` method, which + is the only method declated in the + :py:class:`~ckan.plugins.interfaces.IConfigurer` interface. CKAN will call + this method when it starts up, to give our plugin a chance to modify CKAN's + configuration settings. + +4. Finally, our + :py:meth:`~ckanext.example_theme.plugin_v2.ExampleThemePlugin.update_config` + method calls :py:func:`~ckan.plugins.toolkit.add_template_directory` to + register its custom template directory with CKAN: + + .. literalinclude:: ../ckanext/example_theme/plugin_v2.py + :start-after: # that CKAN will use this plugin's custom templates. + + This tells CKAN to look for template files in the + ``ckanext-example_theme/ckanext/example_theme/templates/`` whenever it + renders a page. Any template file in this directory that has the same name + as one of CKAN's default template files, will be used instead of the default + file. + +Now, let's customize the CKAN front page. Create the +``ckanext-example_theme/ckanext/example_theme/templates/`` directory, create a +``home`` directory inside the ``templates`` directory, and create an empty +``index.html`` file inside the ``home`` directory:: + + ckanext-example_theme/ + ckanext/ + example_theme/ + templates/ + home/ + index.html <-- An empty file. + +Restart the development server (``paster serve development.ini``), and open the +CKAN front page (`127.0.0.1:5000 `_ by default) in your +web browser. You should see an empty page, because we've replaced the template +file for the front page with an empty file. + + +Extending default templates with ``{% ckan_extends %}`` +------------------------------------------------------- + +CKAN template files are written in the `Jinja2`_ templating language. Jinja +template files, such as our ``index.html`` file, are simply text files that, +when processed, generate any text-based output format such as ``HTML``, +``XML``, ``CSV``, etc. Most of the templates file in CKAN generate ``HTML``. + +In Jinja templates snippets of text like ``{% ... %}`` are +Jinja tags that control the logic of the template. For example, CKAN provides +a custom Jinja tag ``{% ckan_extends %}`` that we can use to declare that our +``home/index.html`` template extends the default ``home/index.html`` template. +Edit the empty ``index.html`` file you just created, and add one line: + +.. literalinclude:: ../ckanext/example_theme/v3/templates/home/index.html + +If you now restart the development server and reload the CKAN front page in +your browser, you should see the normal front page appear again. When CKAN +processes our ``index.html`` file, the ``{% ckan_extends %}`` tag tells it to +process the default ``index.html`` file first. + + +Replacing template blocks with ``{% block %}`` +---------------------------------------------- + +Jinja templates can contain *blocks* that child templates can override. For +example, CKAN's default ``home/index.html`` template has a block that contains +the Jinja and HTML code for the "featured groups" that appear on the front page +by default:: + + {% block secondary_content %} +
+ {% for group in c.group_package_stuff %} +
+
+ {% snippet 'snippets/group_item.html', group=group.group_dict, truncate=50, truncate_title=35 %} +
+
+ {% endfor %} +
+ {% endblock %} .. todo:: - * Introduce Fanstatic - * Use the plugin to register a Fanstatic library - * Use ``{% resource %}`` to load stuff from the library - * Presumably you can also load stuff from the core library? + Fix ``c.group_package_stuff`` above (stupid name). +.. todo:: Fix the line wrapping in the code sample above ----------------------- -Customizing CKAN's CSS ----------------------- +.. todo:: Insert screenshot of the part of the page that this template renders? + +When a custom template file extends one of CKAN's default template files using +``{% ckan_extends %}``, it can replace any of the blocks from the default +template with its own code by using ``{% block %}``. Edit your ``index.html`` +file again and change the contents to: + +.. literalinclude:: ../ckanext/example_theme/v4/templates/home/index.html + +Restart the development server, and reload the CKAN front page in your browser. +You should see that the featured groups section of the page has been replaced, +but the rest of the page remains intact. + +.. topic:: Extending parent blocks with Jinja's ``{{ super() }}`` + + If you want to add some code to a block but don't want to replace the entire + block, you can use Jinja's ``{{ super() }}`` tag:: + + {% ckan_extends %} + + {% block secondary_content %} + {{ super() }} + Hello block world! + {% endblock %} + + When the child block above is rendered, Jinja will replace the + ``{{ super() }}`` tag with the contents of the parent block. + + .. todo:: Make the ``super()`` example above into a proper included example. + +.. todo:: Need something here about what variables are available to templates: + c, h, g. etc. plus anything explicitly passed in by the controller or parent + template. + +Template helper functions +------------------------- + +Now let's put some interesting content into our custom template block. +One way for templates to get content out of CKAN is by calling CKAN's +*template helper functions*. + +For example, let's replace the featured groups on the front page with an +activity stream of the site's recently created, updated and deleted datasets. +Change the code in ``index.html`` to this: + +.. literalinclude:: ../ckanext/example_theme/v6/templates/home/index.html + +Reload the CKAN front page in your browser (it shouldn't be necessary to +restart the web server, if you've only made changes to template files) and +you should see a new activity stream on the front page. + +To call a template helper function we use a Jinja2 *expression* (code wrapped +in ``{{ ... }}`` brackets), and we use the global variable ``h`` (available +to all templates) to access the helper: + +.. literalinclude:: ../ckanext/example_theme/v6/templates/home/index.html + :start-after: {% block secondary_content %} + :end-before: {% endblock %} + +To see what other template helper functions are available, look at the +:doc:`template helper functions reference docs `. + + +Adding your own template helper functions +----------------------------------------- + +Plugins can add their own template helper functions by implementing CKAN's +:py:class:`~ckan.plugins.interfaces.ITemplateHelpers` plugin interface. +(see :doc:`writing-extensions` for a detailed explanation of CKAN plugins and +plugin interfaces). + +Let's add another item to our custom front page: a "dataset of the day". We'll +add a custom template helper function to select the dataset to be shown. +First, in our ``plugin.py`` file we need to implement +:py:class:`~ckan.plugins.interfaces.ITemplateHelpers` and provide our helper +function. Change the contents of +``ckanext-example_theme/ckanext/example_theme/plugin.py`` to look like this: + +.. literalinclude:: ../ckanext/example_theme/v7/plugin.py + +.. todo:: Explain exactly what the new lines above do. + + Mention why the helper is named as it is. + +Now that we've registered our helper function, we need to call it from our +template. As with CKAN's default template helpers, templates access custom +helpers via the global variable ``h``. +Edit ``ckanext-example_theme/ckanext/example_theme/home/index.html`` to look +like this: + +.. literalinclude:: ../ckanext/example_theme/v7/templates/home/index.html + +Now restart your web server and reload your CKAN front page in your browser. +You should see the name of a random dataset appear on the page, and each time +you reload the page you'll get a different name. + +Simply displaying the name of a dataset isn't very good. We want to show the +dataset's title not its name, have the title be hyperlinked to the dataset's +page, and also show some other information about the dataset such as its notes +and file formats. To display our dataset of the day nicely, we'll use CKAN's +*template snippets*. + + +Template snippets +----------------- + +*Template snippets* are small snippets of template code that, just like helper +functions, can be called from any template file. To call a snippet, you use +another of CKAN's custom Jinja2 tags: ``{% snippet %}``. CKAN comes with a +selection of snippets, which you can find in the various ``snippets`` +directories in ``ckan/templates/``, such as ``ckan/templates/snippets/`` +and ``ckan/templates/package/snippets/``. .. todo:: - * Introduce CSS? - * Use Fanstatic to add a CSS file - * Use Bootstrap's CSS files and CKAN core's - * See the CKAN style guide - * custom.less: Edit ckan/public/base/less/custom.less and then run ./bin/less - or ./bin/less --production + Autodoc all the default snippets, link to reference docs. +For example, ``ckan/templates/snippets/package_item.html`` is a snippet that +renders a dataset nicely. The default CKAN templates use this snippet whenever +they want to show a list of packages, for example on the datasets page, +a group's page, an organization's page or a user's page: ------------------------------ -Customizing CKAN's JavaScript ------------------------------ +.. literalinclude:: ../ckan/templates/snippets/package_item.html + :end-before: #} + +Let's change our ``index.html`` file to call this snippet: + +.. literalinclude:: ../ckanext/example_theme/v8/templates/home/index.html + +The ``{% snippet %}`` tag takes one or more arguments. The first argument is +the name of the snippet to call. Any further arguments will be passed into +the snippet as parameters. As in the ``package_item.html`` docstring above, +each snippet's docstring should document the parameters it requires. In this +example was pass just one parameter to the snippet: the dataset to be rendered. + +If you reload your CKAN front page in your web browser now, you should see the +dataset of the day rendered nicely. + + +Adding your own template snippets +--------------------------------- + +Just as plugins can add their own template helper functions, they can also add +their own snippets. To add template snippets, all a plugin needs to do is add a +``snippets`` directory in its ``templates`` directory, and start adding files. +The snippets will be callable from other templates immediately. + +Let's add a custom snippet to change how our dataset of the day is displayed. +Create a new directory ``ckanext-example_theme/templates/snippets/`` containing +a file named ``example_theme_dataset_of_the_day.html`` with these contents: + +.. literalinclude:: ../ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html + +Now edit your ``index.html`` file and change to use our new snippet: + +.. literalinclude:: ../ckanext/example_theme/v9/templates/home/index.html + +Restart your web server and reload your CKAN front page in your browser, and +you should see the display of the dataset of the day change. .. todo:: - * How to load JavaScript modules - * jQuery - * Bootstrap's JavaScript stuff - * Other stuff in javascript-module-tutorial.rst + Make the snippet better, and explain the snippet line-by-line. +.. warning:: Snippet overriding + If a plugin adds a snippet with the same name as one of CKAN's default + snippets, the plugin's snippet will override the default snippet wherever + the default snippet is used. + Also if two plugins both have snippets with the same name, one of the + snippets will override the other. <-- TODO: Verify whether this is true + .. todo:: ----- + Exactly what order are ``snippets`` directories read in, and what + overrides what? -Create Custom Extension ------------------------ +.. note:: -This method is best for you want to customize the HTML templates of you CKAN -instance. It's also more extensible and means you can make sure you keep your -custom theme as seperate from CKAN core as possible. + Snippets don't have access to the global template context, so global + variables such as ``c``, ``h`` and ``g`` that are available to templates + are not available to snippets. -Here follows the main topics you'll need in order to understand how to write -a custom extension in order to customize your CKAN instance. + The only variables available to snippets are those explicitly passed into + the snippet by the parent template, when it calls the snippet using a + ``{% snippet %}`` tag. + Keeping snippets "modular" in this way makes debugging template problems + much easier. -Customizing the HTML -~~~~~~~~~~~~~~~~~~~~ + .. todo:: Verify whether this is completely true. -The main templates within CKAN use the templating language `Jinja2`_. Jinja2 -has template inheritance which means that you don't have to re-write a whole -template in order to change small elements within templates. -For more information on how to exactly change the HTML of your CKAN instance: -please read the `Templating > Templating within extensions`_ documentation. +Bootstrap +--------- -Including custom Stylesheets, JavaScript and images -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Jinja2 basics +------------- -Within CKAN we use a resource manager to handle the static resources that are -required by any given template. In order to include a stylesheet or a -JavaScript document you should tell the resource manager of its existence and -then include it within your template. -For more information on how resources work within CKAN and how to add custom -resources to your extension: please read the -`Resources > Resources within extensions`_ documentation. +.. Link to reference docs for CKAN's custom Jinja2 tags and form macros. -.. Note:: - The main CKAN theme is a heavily customized version of `Bootstrap`_. - However the core of Bootstrap is no different in CKAN and therefore people - familiar with Bootstrap should feel right at home writing custom HTML and - CSS for CKAN. +CKAN custom Jinja2 tags reference +--------------------------------- -Customizing the JavaScript -~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------------------------------------------------- +Adding CSS, JavaScript, images and other static files using Fanstatic +--------------------------------------------------------------------- + +.. todo:: + + * Introduce Fanstatic + * Use the plugin to register a Fanstatic library + * Use ``{% resource %}`` to load stuff from the library + * Presumably you can also load stuff from the core library? -Within CKAN core we have a concept of JavaScript modules which allow you to -simply attach JavaScript to DOM elements via HTML5 data attributes. -For more information on what a JavaScript module is and how to build one: -please read the `Building a JavaScript Module`_ documentation. +---------------------- +Customizing CKAN's CSS +---------------------- +.. Use the styles block in base.html to add a global css file -Customizing the CSS -~~~~~~~~~~~~~~~~~~~ +.. todo:: -To customize your CSS all you really need to know is how to add a stylesheet as -a resource. Beyond that it's purely writing your own CSS and making sure it's -included on the correct pages. + * Introduce CSS? + * Use Fanstatic to add a CSS file + * Use Bootstrap's CSS files and CKAN core's + * See the CKAN style guide + * custom.less: Edit ckan/public/base/less/custom.less and then run ./bin/less + or ./bin/less --production -For more information on how CSS works in CKAN core: please read the -`Front End Documentation > Stylesheets`_ documentation. -.. Note:: - In CKAN core we use `LESS`_ to pre-process our main CSS document. We do - this to make the core CSS more maintainable (as well as to offer different - basic colour styles on our default theme). It's not necessary that you do - the same, but we'd recommend using something like it if you plan on - customizing your CKAN instance heavily. +----------------------------- +Customizing CKAN's JavaScript +----------------------------- +.. Use the scripts block in base.html to add a global javascript file -.. _Bootstrap: http://getbootstrap.com/ -.. _Jinja2: http://Jinja2.pocoo.org/ -.. _markdown: http://daringfireball.net/projects/markdown/ -.. _LESS: http://lesscss.org/ -.. _Templating > Templating within extensions: ./templating.html#templating-within-extensions -.. _Resources > Resources within extensions: ./resources.html#resources-within-extensions -.. _Building a JavaScript Module: ./javascript-module-tutorial.html -.. _Front End Documentation > Stylesheets: ./frontend-development.html#stylesheets -.. _CKAN Configuration Options > Front-End Settings: ./configuration.html#front-end-settings -.. _CKAN Configuration Options > Theming Settings: ./configuration.html#theming-settings +.. todo:: + * How to load JavaScript modules + * jQuery + * Bootstrap's JavaScript stuff + * Other stuff in javascript-module-tutorial.rst diff --git a/setup.py b/setup.py index 8588da6a0e1..832c203761b 100644 --- a/setup.py +++ b/setup.py @@ -122,7 +122,14 @@ recline_preview=ckanext.reclinepreview.plugin:ReclinePreview example_itemplatehelpers=ckanext.example_itemplatehelpers.plugin:ExampleITemplateHelpersPlugin example_idatasetform=ckanext.example_idatasetform.plugin:ExampleIDatasetFormPlugin - example_theme=ckanext.example_theme.plugin:ExampleThemePlugin + example_theme_v1=ckanext.example_theme.v1.plugin:ExampleThemePlugin + example_theme_v2=ckanext.example_theme.v2.plugin:ExampleThemePlugin + example_theme_v3=ckanext.example_theme.v3.plugin:ExampleThemePlugin + example_theme_v4=ckanext.example_theme.v4.plugin:ExampleThemePlugin + example_theme_v6=ckanext.example_theme.v6.plugin:ExampleThemePlugin + example_theme_v7=ckanext.example_theme.v7.plugin:ExampleThemePlugin + example_theme_v8=ckanext.example_theme.v8.plugin:ExampleThemePlugin + example_theme_v9=ckanext.example_theme.v9.plugin:ExampleThemePlugin [ckan.system_plugins] domain_object_mods = ckan.model.modification:DomainObjectModificationExtension From 751871f467b84b41918fc7ae0c79f96d755c8239 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 4 Sep 2013 17:33:50 +0200 Subject: [PATCH 018/395] [#847] Remove a TODO that doesn't belong --- doc/theming.rst | 5 ----- 1 file changed, 5 deletions(-) diff --git a/doc/theming.rst b/doc/theming.rst index c129fe31632..c817ac1bb4f 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -4,11 +4,6 @@ Theming ======= -.. todo:: - - Add more to :doc:`getting-started`, is there more that can be done in the - config file, e.g. site logo? - The CKAN frontend can be fully customized by developing a CKAN theme. If you just want to do some simple customizations such as changing the title of your CKAN site, or making some small CSS customizations, From 0cbea1d1e35aa310a7aad724e97838791234c8dc Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 4 Sep 2013 18:41:04 +0200 Subject: [PATCH 019/395] [#847] Docs: add ckanext-example_theme Sphinx substitution --- doc/theming.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/doc/theming.rst b/doc/theming.rst index c817ac1bb4f..268888e3c74 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -1,5 +1,7 @@ .. _Jinja2: http://jinja.pocoo.org/ +.. |extension_dir| replace:: ckanext-example_theme + ======= Theming ======= @@ -44,14 +46,14 @@ an extension and plugin. For a detailed explanation of the steps below, see |activate| cd |virtualenv|/src - paster --plugin=ckan create -t ckanext ckanext-example_theme + paster --plugin=ckan create -t ckanext |extension_dir| -2. Create the file ``ckanext-example_theme/ckanext/example_theme/plugin.py`` +2. Create the file ``|extension_dir|/ckanext/example_theme/plugin.py`` with the following contents: .. literalinclude:: ../ckanext/example_theme/v1/plugin.py -3. Edit the ``entry_points`` in ``ckanext-example_theme/setup.py``:: +3. Edit the ``entry_points`` in ``|extension_dir|/setup.py``:: entry_points=''' [ckan.plugins] @@ -63,7 +65,7 @@ an extension and plugin. For a detailed explanation of the steps below, see .. parsed-literal:: |activate| - cd |virtualenv|/src/ckanext-example_theme + cd |virtualenv|/src/|extension_dir| python setup.py develop 5. Add the plugin to the ``ckan.plugins`` setting in your |development.ini| @@ -145,17 +147,17 @@ This new code does a few things: :start-after: # that CKAN will use this plugin's custom templates. This tells CKAN to look for template files in the - ``ckanext-example_theme/ckanext/example_theme/templates/`` whenever it + ``|extension_dir|/ckanext/example_theme/templates/`` whenever it renders a page. Any template file in this directory that has the same name as one of CKAN's default template files, will be used instead of the default file. Now, let's customize the CKAN front page. Create the -``ckanext-example_theme/ckanext/example_theme/templates/`` directory, create a +``|extension_dir|/ckanext/example_theme/templates/`` directory, create a ``home`` directory inside the ``templates`` directory, and create an empty ``index.html`` file inside the ``home`` directory:: - ckanext-example_theme/ + |extension_dir|/ ckanext/ example_theme/ templates/ @@ -292,7 +294,7 @@ add a custom template helper function to select the dataset to be shown. First, in our ``plugin.py`` file we need to implement :py:class:`~ckan.plugins.interfaces.ITemplateHelpers` and provide our helper function. Change the contents of -``ckanext-example_theme/ckanext/example_theme/plugin.py`` to look like this: +``|extension_dir|/ckanext/example_theme/plugin.py`` to look like this: .. literalinclude:: ../ckanext/example_theme/v7/plugin.py @@ -303,7 +305,7 @@ function. Change the contents of Now that we've registered our helper function, we need to call it from our template. As with CKAN's default template helpers, templates access custom helpers via the global variable ``h``. -Edit ``ckanext-example_theme/ckanext/example_theme/home/index.html`` to look +Edit ``|extension_dir|/ckanext/example_theme/home/index.html`` to look like this: .. literalinclude:: ../ckanext/example_theme/v7/templates/home/index.html @@ -364,7 +366,7 @@ their own snippets. To add template snippets, all a plugin needs to do is add a The snippets will be callable from other templates immediately. Let's add a custom snippet to change how our dataset of the day is displayed. -Create a new directory ``ckanext-example_theme/templates/snippets/`` containing +Create a new directory ``|extension_dir|/templates/snippets/`` containing a file named ``example_theme_dataset_of_the_day.html`` with these contents: .. literalinclude:: ../ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html From 910ca29a902b265e5788864f533a70d8626cbcf1 Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:14 +0530 Subject: [PATCH 020/395] New option to disable user creation via web --- ckan/config/deployment.ini_tmpl | 1 + ckan/logic/auth/create.py | 22 +++++++++++++++------- ckan/new_authz.py | 2 ++ ckan/templates/header.html | 4 +++- ckan/templates/user/login.html | 20 +++++++++++--------- test-core.ini | 3 ++- 6 files changed, 34 insertions(+), 18 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index 97c4f9412ba..6a0833d2d9f 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -66,6 +66,7 @@ ckan.auth.user_create_organizations = true ckan.auth.user_delete_groups = true ckan.auth.user_delete_organizations = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ## Search Settings diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index e96b789bcf5..35a9f39d876 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -103,14 +103,22 @@ def rating_create(context, data_dict): # No authz check in the logic function return {'success': True} -def user_create(context, data_dict=None): - user = context['user'] - if ('api_version' in context - and not new_authz.check_config_permission('create_user_via_api')): - return {'success': False, 'msg': _('User %s not authorized to create users') % user} - else: - return {'success': True} +def user_create(context, data_dict=None): + # create_user_via_api is deprecated + using_api = 'api_version' in context + create_user_via_api = new_authz.check_config_permission( + 'create_user_via_api') + create_user_via_web = new_authz.check_config_permission( + 'create_user_via_web') + + if using_api and not create_user_via_api: + return {'success': False, 'msg': _('User {user} not authorized to ' + 'create users via the API').format(user=context.get('user'))} + if not using_api and not create_user_via_web: + return {'success': False, 'msg': _('Not authorized to ' + 'create users')} + return {'success': True} def _check_group_auth(context, data_dict): diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 00a7dfe0e17..03ca209faf4 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -86,6 +86,7 @@ def _build(self): def clear_auth_functions_cache(): _AuthFunctions.clear() + CONFIG_PERMISSIONS.clear() def auth_functions_list(): @@ -319,6 +320,7 @@ def get_user_id_for_username(user_name, allow_none=False): 'user_delete_groups': True, 'user_delete_organizations': True, 'create_user_via_api': False, + 'create_user_via_web': True, } CONFIG_PERMISSIONS = {} diff --git a/ckan/templates/header.html b/ckan/templates/header.html index 5b4ebe225a7..268b370e968 100644 --- a/ckan/templates/header.html +++ b/ckan/templates/header.html @@ -49,7 +49,9 @@
    {% block header_account_notlogged %}
  • {% link_for _('Log in'), controller='user', action='login' %}
    • -
  • {% link_for _('Register'), controller='user', action='register', class_='sub' %}
    • + {% if h.check_access('user_create') %} +
  • {% link_for _('Register'), controller='user', action='register', class_='sub' %}
    • + {% endif %} {% endblock %}
diff --git a/ckan/templates/user/login.html b/ckan/templates/user/login.html index c84e3522748..98c99510570 100644 --- a/ckan/templates/user/login.html +++ b/ckan/templates/user/login.html @@ -18,15 +18,17 @@

{% block page_heading %}{{ _('Login') }}{% endblock %}< {% endblock %} {% block secondary_content %} -
-

{{ _('Need an Account?') }}

-
-

{% trans %}Then sign right up, it only takes a minute.{% endtrans %}

-

- {{ _('Create an Account') }} -

-
-
+ {% if h.check_access('user_create') %} +
+

{{ _('Need an Account?') }}

+
+

{% trans %}Then sign right up, it only takes a minute.{% endtrans %}

+

+ {{ _('Create an Account') }} +

+
+
+ {% endif %}

{{ _('Forgotten your details?') }}

diff --git a/test-core.ini b/test-core.ini index 21dc08cfae9..08aba34ed24 100644 --- a/test-core.ini +++ b/test-core.ini @@ -31,6 +31,7 @@ solr_url = http://127.0.0.1:8983/solr ckan.auth.user_create_organizations = true ckan.auth.user_create_groups = true ckan.auth.create_user_via_api = false +ckan.auth.create_user_via_web = true ckan.auth.create_dataset_if_not_in_organization = true ckan.auth.anon_create_dataset = false ckan.auth.user_delete_groups=true @@ -80,7 +81,7 @@ smtp.mail_from = info@test.ckan.net ckan.locale_default = en ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv -ckan.locales_filtered_out = +ckan.locales_filtered_out = ckan.datastore.enabled = 1 From 6bc0c3ee3b9ea158ad358c5697e26eedd372ba9f Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 14:59:50 +0530 Subject: [PATCH 021/395] Add documentation for new config option --- doc/configuration.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/doc/configuration.rst b/doc/configuration.rst index 0efe9573226..853ffd23939 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -355,6 +355,20 @@ Default value: ``False`` Allow new user accounts to be created via the API. +.. _ckan.auth.create_user_via_web: + +ckan.auth.create_user_via_api +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.auth.create_user_via_web = True + +Default value: ``True`` + + +Allow new user accounts to be created via the Web. + .. end_config-authorization From c75e487ff99765418d5c82e9f683e2d694afc98d Mon Sep 17 00:00:00 2001 From: Nigel Babu Date: Thu, 5 Sep 2013 16:46:10 +0530 Subject: [PATCH 022/395] Fix docs, remove deprecated comment --- ckan/logic/auth/create.py | 1 - doc/configuration.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index 35a9f39d876..ec38483bd0d 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -105,7 +105,6 @@ def rating_create(context, data_dict): def user_create(context, data_dict=None): - # create_user_via_api is deprecated using_api = 'api_version' in context create_user_via_api = new_authz.check_config_permission( 'create_user_via_api') diff --git a/doc/configuration.rst b/doc/configuration.rst index 853ffd23939..29ecbd310d8 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -357,7 +357,7 @@ Allow new user accounts to be created via the API. .. _ckan.auth.create_user_via_web: -ckan.auth.create_user_via_api +ckan.auth.create_user_via_web ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example:: From 7ce1555ca3dc65adbb882a2373d4f99ce65fbf8c Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 6 Sep 2013 16:19:27 +0100 Subject: [PATCH 023/395] [#1038] Fix up authz config error and deleted groups * Fix - CONFIG_PERMISSIONS were not read properly following my prev changes * Fix deleted child groups being visible. * Fix up a couple of tests --- ckan/logic/auth/create.py | 7 +++++++ ckan/model/group.py | 1 + ckan/new_authz.py | 2 +- ckan/tests/functional/test_group.py | 18 +++++++++++++----- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ckan/logic/auth/create.py b/ckan/logic/auth/create.py index ae98e257914..6061e4eb69b 100644 --- a/ckan/logic/auth/create.py +++ b/ckan/logic/auth/create.py @@ -114,6 +114,13 @@ def user_create(context, data_dict=None): def _check_group_auth(context, data_dict): + '''Has this user got update permission for all of the given groups? + If there is a package in the context then ignore that package's groups. + :returns: False if not allowed to update one (or more) of the given groups. + True otherwise. i.e. True is the default. A blank data_dict + mentions no groups, so it returns True. + + ''' # FIXME This code is shared amoung other logic.auth files and should be # somewhere better if not data_dict: diff --git a/ckan/model/group.py b/ckan/model/group.py index f21bafb3cb8..a25cebf9927 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -177,6 +177,7 @@ def get_children_groups(self, type='group'): # actually did, but is now far simpler. results = meta.Session.query(Group.id, Group.name, Group.title).\ filter_by(type=type).\ + filter_by(state='active').\ join(Member, Member.table_id == Group.id).\ filter_by(group=self).\ filter_by(table_name='group').\ diff --git a/ckan/new_authz.py b/ckan/new_authz.py index a8f13063f60..370b19fb581 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -354,7 +354,7 @@ def check_config_permission(permission): key = 'ckan.auth.' + perm default = CONFIG_PERMISSIONS_DEFAULTS[perm] CONFIG_PERMISSIONS[perm] = config.get(key, default) - if isinstance(perm, bool): + if isinstance(default, bool): CONFIG_PERMISSIONS[perm] = asbool(CONFIG_PERMISSIONS[perm]) if permission in CONFIG_PERMISSIONS: return CONFIG_PERMISSIONS[permission] diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index b07bbe7ea6f..4f4fddc3e05 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -24,6 +24,10 @@ def setup_class(self): model.Session.remove() CreateTestData.create() + # reduce extraneous logging + from ckan.lib import activity_streams_session_extension + activity_streams_session_extension.logger.level = 100 + @classmethod def teardown_class(self): model.repo.rebuild_db() @@ -102,17 +106,21 @@ def test_children(self): def test_sorting(self): model.repo.rebuild_db() + testsysadmin = model.User(name=u'testsysadmin') + testsysadmin.sysadmin = True + model.Session.add(testsysadmin) + pkg1 = model.Package(name="pkg1") pkg2 = model.Package(name="pkg2") model.Session.add(pkg1) model.Session.add(pkg2) CreateTestData.create_groups([{'name': "alpha", 'packages': []}, - {'name': "beta", - 'packages': ["pkg1", "pkg2"]}, - {'name': "delta", - 'packages': ["pkg1"]}, - {'name': "gamma", 'packages': []}], + {'name': "beta", + 'packages': ["pkg1", "pkg2"]}, + {'name': "delta", + 'packages': ["pkg1"]}, + {'name': "gamma", 'packages': []}], admin_user_name='testsysadmin') context = {'model': model, 'session': model.Session, From f7f5049b0ab0a7220210722461bf56768d33df89 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 6 Sep 2013 16:42:30 +0000 Subject: [PATCH 024/395] [#1038] PEP8 fixes only. --- ckan/lib/create_test_data.py | 47 +++++++++++++++----------- ckan/logic/validators.py | 5 +-- ckan/model/group.py | 51 ++++++++++++++++------------- ckan/new_authz.py | 14 +++++--- ckan/tests/functional/test_group.py | 14 ++++---- ckan/tests/test_coding_standards.py | 1 - 6 files changed, 75 insertions(+), 57 deletions(-) diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 80f32a1038d..2ca59a42fcc 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -221,19 +221,23 @@ def create_arbitrary(cls, package_dicts, relationships=[], group = model.Group.by_name(unicode(group_name)) if not group: if not group_name in new_groups: - group = model.Group(name=unicode(group_name)) + group = model.Group(name= + unicode(group_name)) model.Session.add(group) new_group_names.add(group_name) new_groups[group_name] = group else: - # If adding multiple packages with the same group name, - # model.Group.by_name will not find the group as the - # session has not yet been committed at this point. - # Fetch from the new_groups dict instead. + # If adding multiple packages with the same + # group name, model.Group.by_name will not + # find the group as the session has not yet + # been committed at this point. Fetch from + # the new_groups dict instead. group = new_groups[group_name] - capacity = 'organization' if group.is_organization \ + capacity = 'organization' if group.is_organization\ else 'public' - member = model.Member(group=group, table_id=pkg.id, table_name='package', capacity=capacity) + member = model.Member(group=group, table_id=pkg.id, + table_name='package', + capacity=capacity) model.Session.add(member) if group.is_organization: pkg.owner_org = group.id @@ -338,8 +342,8 @@ def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): 'type', 'is_organization')) for group_dict in group_dicts: if model.Group.by_name(unicode(group_dict['name'])): - log.warning('Cannot create group "%s" as it already exists.' % \ - (group_dict['name'])) + log.warning('Cannot create group "%s" as it already exists.' % + group_dict['name']) continue pkg_names = group_dict.pop('packages', []) group = model.Group(name=unicode(group_dict['name'])) @@ -353,18 +357,19 @@ def create_groups(cls, group_dicts, admin_user_name=None, auth_profile=""): for pkg_name in pkg_names: pkg = model.Package.by_name(unicode(pkg_name)) assert pkg, pkg_name - member = model.Member(group=group, table_id=pkg.id, table_name='package') + member = model.Member(group=group, table_id=pkg.id, + table_name='package') model.Session.add(member) model.Session.add(group) - admins = [model.User.by_name(user_name) \ - for user_name in group_dict.get('admins', [])] \ - + admin_users + admins = [model.User.by_name(user_name) + for user_name in group_dict.get('admins', [])] + \ + admin_users for admin in admins: member = model.Member(group=group, table_id=admin.id, table_name='user', capacity='admin') model.Session.add(member) - editors = [model.User.by_name(user_name) \ - for user_name in group_dict.get('editors', [])] + editors = [model.User.by_name(user_name) + for user_name in group_dict.get('editors', [])] for editor in editors: member = model.Member(group=group, table_id=editor.id, table_name='user', capacity='editor') @@ -400,7 +405,8 @@ def create(cls, auth_profile="", package_type=None): * Associated tags, etc etc ''' if auth_profile == "publisher": - organization_group = model.Group(name=u"organization_group", type="organization") + organization_group = model.Group(name=u"organization_group", + type="organization") cls.pkg_names = [u'annakarenina', u'warandpeace'] pkg1 = model.Package(name=cls.pkg_names[0], type=package_type) @@ -525,7 +531,6 @@ def create(cls, auth_profile="", package_type=None): model.repo.commit_and_remove() - # method used in DGU and all good tests elsewhere @classmethod def create_users(cls, user_dicts): @@ -539,9 +544,11 @@ def create_users(cls, user_dicts): @classmethod def _create_user_without_commit(cls, name='', **user_dict): - if model.User.by_name(name) or (user_dict.get('open_id') and model.User.by_openid(user_dict.get('openid'))): - log.warning('Cannot create user "%s" as it already exists.' % \ - (name or user_dict['name'])) + if model.User.by_name(name) or \ + (user_dict.get('open_id') and + model.User.by_openid(user_dict.get('openid'))): + log.warning('Cannot create user "%s" as it already exists.' % + name or user_dict['name']) return # User objects are not revisioned so no need to create a revision user_ref = name or user_dict['openid'] diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 0fb3887798e..429fcb5a774 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -36,8 +36,9 @@ def owner_org_validator(key, data, errors, context): group_id = group.id user = context['user'] user = model.User.get(user) - if not(user.sysadmin or new_authz.has_user_permission_for_group_or_org( - group_id, user.name, 'create_dataset')): + if not(user.sysadmin or + new_authz.has_user_permission_for_group_or_org( + group_id, user.name, 'create_dataset')): raise Invalid(_('You cannot add a dataset to this organization')) data[key] = group_id diff --git a/ckan/model/group.py b/ckan/model/group.py index a25cebf9927..ee637dde82f 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -103,15 +103,19 @@ def related_packages(self): def __unicode__(self): # refer to objects by name, not ID, to help debugging if self.table_name == 'package': - table_info = 'package=%s' % meta.Session.query(_package.Package).get(self.table_id).name + table_info = 'package=%s' % meta.Session.query(_package.Package).\ + get(self.table_id).name elif self.table_name == 'group': - table_info = 'group=%s' % meta.Session.query(Group).get(self.table_id).name + table_info = 'group=%s' % meta.Session.query(Group).\ + get(self.table_id).name else: - table_info = 'table_name=%s table_id=%s' % (self.table_name, self.table_id) + table_info = 'table_name=%s table_id=%s' % (self.table_name, + self.table_id) return u'' % \ (self.group.name if self.group else repr(self.group), table_info, self.capacity, self.state) + class Group(vdm.sqlalchemy.RevisionedObjectMixin, vdm.sqlalchemy.StatefulObjectMixin, domain_object.DomainObject): @@ -172,10 +176,10 @@ def get_children_groups(self, type='group'): '''Returns the groups one level underneath this group in the hierarchy. Groups come in a list of dicts, each keyed by "id", "name" and "title". ''' - # The original intention of this method was to provide the full depth of - # the tree, but the CTE was incorrect. This new query does what that old CTE - # actually did, but is now far simpler. - results = meta.Session.query(Group.id, Group.name, Group.title).\ + # The original intention of this method was to provide the full depth + # of the tree, but the CTE was incorrect. This new query does what that + # old CTE actually did, but is now far simpler. + results = meta.Session.query(Group.id, Group.name, Group.title).\ filter_by(type=type).\ filter_by(state='active').\ join(Member, Member.table_id == Group.id).\ @@ -184,12 +188,13 @@ def get_children_groups(self, type='group'): filter_by(state='active').\ all() - return [{'id':id_, 'name': name, 'title': title} + return [{'id': id_, 'name': name, 'title': title} for id_, name, title in results] def get_children_group_hierarchy(self, type='group'): - '''Returns the groups in all levels underneath this group in the hierarchy. - The ordering is such that children always come after their parent. + '''Returns the groups in all levels underneath this group in the + hierarchy. The ordering is such that children always come after their + parent. :rtype: a list of tuples, each one a Group and the ID its their parent group. @@ -200,14 +205,16 @@ def get_children_group_hierarchy(self, type='group'): (, u'd2e25b41-720c-4ba7-bc8f-bb34b185b3dd')] ''' results = meta.Session.query(Group, 'parent_id').\ - from_statement(HIERARCHY_DOWNWARDS_CTE).params(id=self.id, type=type).all() + from_statement(HIERARCHY_DOWNWARDS_CTE).\ + params(id=self.id, type=type).all() return results def get_parent_group_hierarchy(self, type='group'): - '''Returns this group's parent, parent's parent, parent's parent's parent - etc.. Sorted with the top level parent first.''' + '''Returns this group's parent, parent's parent, parent's parent's + parent etc.. Sorted with the top level parent first.''' return meta.Session.query(Group).\ - from_statement(HIERARCHY_UPWARDS_CTE).params(id=self.id, type=type).all() + from_statement(HIERARCHY_UPWARDS_CTE).\ + params(id=self.id, type=type).all() @classmethod def get_top_level_groups(cls, type='group'): @@ -215,12 +222,12 @@ def get_top_level_groups(cls, type='group'): no parent groups. Groups are sorted by title. ''' return meta.Session.query(cls).\ - outerjoin(Member, Member.table_id == Group.id and \ - Member.table_name == 'group' and \ - Member.state == 'active').\ - filter(Member.id==None).\ - filter(Group.type==type).\ - order_by(Group.title).all() + outerjoin(Member, Member.table_id == Group.id and + Member.table_name == 'group' and + Member.state == 'active').\ + filter(Member.id == None).\ + filter(Group.type == type).\ + order_by(Group.title).all() def packages(self, with_private=False, limit=None, return_query=False, context=None): @@ -381,7 +388,6 @@ def __repr__(self): MemberRevision.related_packages = lambda self: [self.continuity.package] - HIERARCHY_DOWNWARDS_CTE = """WITH RECURSIVE child AS ( -- non-recursive term @@ -404,7 +410,8 @@ def __repr__(self): UNION -- recursive term SELECT PG.depth + 1, M.* FROM parenttree PG, public.member M - WHERE PG.group_id = M.table_id AND M.table_name = 'group' AND M.state = 'active' + WHERE PG.group_id = M.table_id AND M.table_name = 'group' + AND M.state = 'active' ) SELECT G.*, PT.depth FROM parenttree AS PT diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 370b19fb581..2f612c82b41 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -213,7 +213,9 @@ def get_roles_with_permission(permission): def has_user_permission_for_group_or_org(group_id, user_name, permission): ''' Check if the user has the given permissions for the group, - allowing for sysadmin rights and permission cascading down groups. ''' + allowing for sysadmin rights and permission cascading down groups. + + ''' if not group_id: return False group = model.Group.get(group_id) @@ -234,15 +236,19 @@ def has_user_permission_for_group_or_org(group_id, user_name, permission): # in the group hierarchy for permission. for capacity in check_config_permission('roles_that_cascade_to_sub_groups'): parent_groups = group.get_parent_group_hierarchy(type=group.type) - group_ids = [group.id for group in parent_groups] + group_ids = [group_.id for group_ in parent_groups] if _has_user_permission_for_groups(user_id, permission, group_ids, capacity=capacity): return True return False -def _has_user_permission_for_groups(user_id, permission, group_ids, capacity=None): + +def _has_user_permission_for_groups(user_id, permission, group_ids, + capacity=None): ''' Check if the user has the given permissions for the particular - group. Can also be filtered by a particular capacity''' + group. Can also be filtered by a particular capacity. + + ''' # get any roles the user has for the group q = model.Session.query(model.Member) \ .filter(model.Member.group_id.in_(group_ids)) \ diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index 4f4fddc3e05..5575bd6e4ba 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -2,7 +2,6 @@ from nose.tools import assert_equal -import ckan.tests.test_plugins as test_plugins import ckan.model as model import ckan.lib.search as search @@ -15,7 +14,6 @@ from ckan.tests import is_search_supported - class TestGroup(FunctionalTestCase): @classmethod @@ -59,14 +57,14 @@ def test_atom_feed_page_negative(self): assert '"page" parameter must be a positive integer' in res, res def test_children(self): - if model.engine_is_sqlite() : + if model.engine_is_sqlite(): from nose import SkipTest raise SkipTest("Can't use CTE for sqlite") group_name = 'deletetest' CreateTestData.create_groups([{'name': group_name, 'packages': []}, - {'name': "parent_group", + {'name': "parent_group", 'packages': []}], admin_user_name='testsysadmin') @@ -155,7 +153,6 @@ def test_sorting(self): assert results[0]['name'] == u'beta', results[0]['name'] assert results[1]['name'] == u'delta', results[1]['name'] - def test_mainmenu(self): # the home page does a package search so have to skip this test if # search is not supported @@ -205,14 +202,16 @@ def test_read_and_authorized_to_edit(self): title = u'Dave\'s books' pkgname = u'warandpeace' offset = url_for(controller='group', action='read', id=name) - res = self.app.get(offset, extra_environ={'REMOTE_USER': 'testsysadmin'}) + res = self.app.get(offset, + extra_environ={'REMOTE_USER': 'testsysadmin'}) assert title in res, res assert 'edit' in res assert name in res def test_new_page(self): offset = url_for(controller='group', action='new') - res = self.app.get(offset, extra_environ={'REMOTE_USER': 'testsysadmin'}) + res = self.app.get(offset, + extra_environ={'REMOTE_USER': 'testsysadmin'}) assert 'Add A Group' in res, res @@ -265,7 +264,6 @@ def setup_class(self): model.Session.add(model.Package(name=self.packagename)) model.repo.commit_and_remove() - @classmethod def teardown_class(self): model.Session.remove() diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index f0b7469c3f4..4481342acd5 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -751,7 +751,6 @@ class TestPep8(object): 'ckan/tests/functional/test_cors.py', 'ckan/tests/functional/test_error.py', 'ckan/tests/functional/test_follow.py', - 'ckan/tests/functional/test_group.py', 'ckan/tests/functional/test_home.py', 'ckan/tests/functional/test_package.py', 'ckan/tests/functional/test_package_relationships.py', From a4e0aaf579eb9649614975f12cf889082802472f Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Sep 2013 14:29:48 +0200 Subject: [PATCH 025/395] [#847] example_theme plugin_v2 tweaks Comments, add a missing import --- ckanext/example_theme/v2/plugin.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckanext/example_theme/v2/plugin.py b/ckanext/example_theme/v2/plugin.py index bf47f41095a..906602627b4 100644 --- a/ckanext/example_theme/v2/plugin.py +++ b/ckanext/example_theme/v2/plugin.py @@ -1,10 +1,15 @@ +'''plugin.py + +''' import ckan.plugins as plugins +import ckan.plugins.toolkit as toolkit class ExampleThemePlugin(plugins.SingletonPlugin): '''An example theme plugin. ''' + # Declare that this class implements IConfigurer. plugins.implements(plugins.IConfigurer) def update_config(self, config): From 6d33afc48d17a63451f3094f26fd1878c1a8f6da Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Sep 2013 14:30:31 +0200 Subject: [PATCH 026/395] [#847] example_theme plugin_v7 tweaks Comments, etc. --- ckanext/example_theme/v7/plugin.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/ckanext/example_theme/v7/plugin.py b/ckanext/example_theme/v7/plugin.py index bfe2a15572a..6cecd83e7c3 100644 --- a/ckanext/example_theme/v7/plugin.py +++ b/ckanext/example_theme/v7/plugin.py @@ -4,14 +4,20 @@ import ckan.plugins.toolkit as toolkit -def example_theme_dataset_of_the_day(): +def dataset_of_the_day(): '''Return the dataset of the day. ''' + # Get a list of the names of all of the site's datasets. dataset_names = toolkit.get_action('package_list')(data_dict={}) + + # Choose one dataset name at random. dataset_name = random.choice(dataset_names) + + # Get the full dictionary object for the chosen dataset. dataset = toolkit.get_action('package_show')( data_dict={'id': dataset_name}) + return dataset @@ -20,6 +26,8 @@ class ExampleThemePlugin(plugins.SingletonPlugin): ''' plugins.implements(plugins.IConfigurer) + + # Declare that this plugin will implement ITemplateHelpers. plugins.implements(plugins.ITemplateHelpers) def update_config(self, config): @@ -29,6 +37,11 @@ def update_config(self, config): toolkit.add_template_directory(config, 'templates') def get_helpers(self): - - return {'example_theme_dataset_of_the_day': - example_theme_dataset_of_the_day} + '''Register the dataset_of_the_day() function above as a template + helper function. + + ''' + # Template helper function names should begin with the name of the + # extension they belong to, to avoid clashing with functions from + # other extensions. + return {'example_theme_dataset_of_the_day': dataset_of_the_day} From e77af5622321212239116b4014ea501600b2165e Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Sep 2013 14:45:08 +0200 Subject: [PATCH 027/395] [#847] Lots of theming docs tweaks Use substitutions for lots of things, and also lots of text tweaks, new stuff about how to find out which template a page uses, global variables, etc. --- doc/theming.rst | 373 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 268 insertions(+), 105 deletions(-) diff --git a/doc/theming.rst b/doc/theming.rst index 268888e3c74..494d5320733 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -1,6 +1,12 @@ .. _Jinja2: http://jinja.pocoo.org/ +.. _CKAN front page: http://127.0.0.1:5000 + +.. |extension_dir| replace:: ``ckanext-example_theme`` +.. |setup.py| replace:: ``ckanext-example_theme/setup.py`` ``ckan.plugins`` +.. |plugin.py| replace:: ``ckanext-example_theme/ckanext/example_theme/plugin.py`` +.. |index.html| replace:: ``ckanext-example_theme/ckanext/example_theme/templates/home/index.html`` +.. |snippets_dir| replace:: ``ckanext-example_theme/ckanext/example_theme/templates/snippets`` -.. |extension_dir| replace:: ckanext-example_theme ======= Theming @@ -11,16 +17,19 @@ If you just want to do some simple customizations such as changing the title of your CKAN site, or making some small CSS customizations, :doc:`getting-started` documents some simple configuration settings you can use. -If you want more control, follow the tutorial below to learn how to develop -your custom CKAN theme. +If you want more control, CKAN themes can customize all aspects of CKAN's +frontend, including changing any of CKAN's templates or template snippets, +adding custom snippets and helper functions, customizing CSS and JavaScript, +and adding static files such as images. Follow the tutorial below to learn how +to develop your custom CKAN theme. ---------------- Theming tutorial ---------------- -This tutorial walks you through the process of creating a CKAN theme, and -demonstrates all of the main features of CKAN theming. +This tutorial walks you through the process of creating an example CKAN theme +that demonstrates all of the main features of CKAN theming. Installing CKAN @@ -48,12 +57,11 @@ an extension and plugin. For a detailed explanation of the steps below, see cd |virtualenv|/src paster --plugin=ckan create -t ckanext |extension_dir| -2. Create the file ``|extension_dir|/ckanext/example_theme/plugin.py`` - with the following contents: +2. Create the file |plugin.py| with the following contents: .. literalinclude:: ../ckanext/example_theme/v1/plugin.py -3. Edit the ``entry_points`` in ``|extension_dir|/setup.py``:: +3. Edit the ``entry_points`` in |setup.py|:: entry_points=''' [ckan.plugins] @@ -77,21 +85,18 @@ an extension and plugin. For a detailed explanation of the steps below, see .. parsed-literal:: - $ paster serve |development.ini| + $ paster serve --reload |development.ini| Starting server in PID 13961. serving on 0.0.0.0:5000 view at http://127.0.0.1:5000 - If your plugin is in the :ref:`ckan.plugins` setting and CKAN starts without - crashing, then your plugin is installed and CKAN can find it. Of course, - your plugin doesn't *do* anything yet. - - -Customizing CKAN's HTML with Jinja2 -=================================== + Open the `CKAN front page`_ in your web browser. If your plugin is in the + :ref:`ckan.plugins` setting and CKAN starts without crashing, then your + plugin is installed and CKAN can find it. Of course, your plugin doesn't + *do* anything yet. -Replacing a default template file ---------------------------------- +Customizing CKAN's HTML with Jinja2 templates +============================================= Every CKAN page is generated by rendering a particular template. For each page of a CKAN site there's a corresponding template file. For example the @@ -100,13 +105,13 @@ front page is generated from the ``ckan/templates/home/index.html`` file, the datasets page at ``/dataset`` is generated from ``ckan/templates/package/search.html``, etc. -.. todo:: - Explain how to find out which template file is used for a given page. +Replacing a default template file +--------------------------------- To customize pages, our plugin needs to register its own custom template directory containing templates file that override the default ones. -Edit the ``plugin.py`` file that we created earlier, so that it looks like +Edit the |plugin.py| file that we created earlier, so that it looks like this: .. literalinclude:: ../ckanext/example_theme/v2/plugin.py @@ -115,7 +120,7 @@ This new code does a few things: 1. It imports CKAN's *plugins toolkit* module: - .. literalinclude:: ../ckanext/example_theme/plugin_v2.py + .. literalinclude:: ../ckanext/example_theme/v2/plugin.py :start-after: import ckan.plugins as plugins :end-before: class ExampleThemePlugin(plugins.SingletonPlugin): @@ -125,8 +130,15 @@ This new code does a few things: 2. It calls :py:func:`~ckan.plugins.core.implements` to declare that it implements the :py:class:`~ckan.plugins.interfaces.IConfigurer` plugin - interface. This tells CKAN that our - :py:class:`~ckanext.example_theme.plugin_v2.ExampleThemePlugin` class + interface: + + .. literalinclude:: ../ckanext/example_theme/v2/plugin.py + :start-after: # Declare that this class implements IConfigurer. + :end-before: def update_config( + + + This tells CKAN that our + :py:class:`~ckanext.example_theme.v2.plugin.ExampleThemePlugin` class implements the methods declared in the :py:class:`~ckan.plugins.interfaces.IConfigurer` interface. CKAN will call these methods of our plugin class at the appropriate times. @@ -134,62 +146,95 @@ This new code does a few things: 3. It implements the :py:meth:`~ckan.plugins.interfaces.IConfigurer.update_config` method, which is the only method declated in the - :py:class:`~ckan.plugins.interfaces.IConfigurer` interface. CKAN will call - this method when it starts up, to give our plugin a chance to modify CKAN's - configuration settings. + :py:class:`~ckan.plugins.interfaces.IConfigurer` interface: + + .. literalinclude:: ../ckanext/example_theme/v2/plugin.py + :pyobject: ExampleThemePlugin.update_config -4. Finally, our - :py:meth:`~ckanext.example_theme.plugin_v2.ExampleThemePlugin.update_config` + CKAN will call this method when it starts up, to give our plugin a chance to + modify CKAN's configuration settings. Our + :py:meth:`~ckanext.example_theme.v2.plugin.ExampleThemePlugin.update_config` method calls :py:func:`~ckan.plugins.toolkit.add_template_directory` to - register its custom template directory with CKAN: + register its custom template directory with CKAN. + This tells CKAN to look for template files in |templates_dir| whenever + it renders a page. Any template file in this directory that has the same + name as one of CKAN's default template files, will be used instead of the + default file. + +Now, let's customize the CKAN front page. We first need to discover which +template file CKAN uses to render the front page, so we can replace it. +Set :ref:`debug` to ``true`` in your |development.ini| file:: + + [DEFAULT] + + # WARNING: *THIS SETTING MUST BE SET TO FALSE ON A PRODUCTION ENVIRONMENT* + debug = true + +Reload the `CKAN front page`_ in your browser, and you should see a *Debug* +link in the footer at the bottom of the page. Click on this link to open the +debug footer. The debug footer displays various information useful for CKAN +frontend development and debugging, including the name of the template file +that was used to render the current page: + +.. todo:: Insert a screenshot of the debug link. + +:: + + Template name: home/index.html + Template path: /usr/lib/ckan/default/src/ckan/ckan/templates/home/index.html + +This tells us that ``home/index.html`` is the main template file used to render +the front page. The debug footer appears at the bottom of every CKAN page, and +can always be used to find the page's template, and other information about the +page. - .. literalinclude:: ../ckanext/example_theme/plugin_v2.py - :start-after: # that CKAN will use this plugin's custom templates. +Now let's override ``home/index.html`` using our plugins' custom ``templates`` +directory. Create the |templates_dir| directory, create a ``home`` directory +inside the ``templates`` directory, and create an empty ``index.html`` file +inside the ``home`` directory: - This tells CKAN to look for template files in the - ``|extension_dir|/ckanext/example_theme/templates/`` whenever it - renders a page. Any template file in this directory that has the same name - as one of CKAN's default template files, will be used instead of the default - file. +.. parsed-literal:: -Now, let's customize the CKAN front page. Create the -``|extension_dir|/ckanext/example_theme/templates/`` directory, create a -``home`` directory inside the ``templates`` directory, and create an empty -``index.html`` file inside the ``home`` directory:: + |extension_dir|/ + ckanext/ + example_theme/ + templates/ + home/ + index.html <-- An empty file. - |extension_dir|/ - ckanext/ - example_theme/ - templates/ - home/ - index.html <-- An empty file. +If you now reload the `CKAN front page`_ in your web browser, you should see +an empty page, because we've replaced the template file for the front page with +an empty file. -Restart the development server (``paster serve development.ini``), and open the -CKAN front page (`127.0.0.1:5000 `_ by default) in your -web browser. You should see an empty page, because we've replaced the template -file for the front page with an empty file. +Jinja2 +------ -Extending default templates with ``{% ckan_extends %}`` -------------------------------------------------------- +.. todo:: Brief introduction to Jinja2 here. CKAN template files are written in the `Jinja2`_ templating language. Jinja template files, such as our ``index.html`` file, are simply text files that, when processed, generate any text-based output format such as ``HTML``, ``XML``, ``CSV``, etc. Most of the templates file in CKAN generate ``HTML``. -In Jinja templates snippets of text like ``{% ... %}`` are -Jinja tags that control the logic of the template. For example, CKAN provides -a custom Jinja tag ``{% ckan_extends %}`` that we can use to declare that our -``home/index.html`` template extends the default ``home/index.html`` template. +In Jinja templates snippets of text like ``{% ... %}`` are Jinja *tags* that +control the logic of the template. + + +Extending templates with ``{% ckan_extends %}`` +----------------------------------------------- + +CKAN provides a custom Jinja tag ``{% ckan_extends %}`` that we can use to +declare that our ``home/index.html`` template extends the default +``home/index.html`` template, instead of completely replacing it. Edit the empty ``index.html`` file you just created, and add one line: .. literalinclude:: ../ckanext/example_theme/v3/templates/home/index.html -If you now restart the development server and reload the CKAN front page in -your browser, you should see the normal front page appear again. When CKAN -processes our ``index.html`` file, the ``{% ckan_extends %}`` tag tells it to -process the default ``index.html`` file first. +If you now reload the `CKAN front page`_ in your browser, you should see the +normal front page appear again. When CKAN processes our ``index.html`` file, +the ``{% ckan_extends %}`` tag tells it to process the default +``home/index.html`` file first. Replacing template blocks with ``{% block %}`` @@ -205,7 +250,8 @@ by default:: {% for group in c.group_package_stuff %}
- {% snippet 'snippets/group_item.html', group=group.group_dict, truncate=50, truncate_title=35 %} + {% snippet 'snippets/group_item.html', group=group.group_dict, + truncate=50, truncate_title=35 %}
{% endfor %} @@ -216,8 +262,6 @@ by default:: Fix ``c.group_package_stuff`` above (stupid name). -.. todo:: Fix the line wrapping in the code sample above - .. todo:: Insert screenshot of the part of the page that this template renders? When a custom template file extends one of CKAN's default template files using @@ -227,30 +271,28 @@ file again and change the contents to: .. literalinclude:: ../ckanext/example_theme/v4/templates/home/index.html -Restart the development server, and reload the CKAN front page in your browser. +Reload the `CKAN front page`_ in your browser. You should see that the featured groups section of the page has been replaced, but the rest of the page remains intact. -.. topic:: Extending parent blocks with Jinja's ``{{ super() }}`` +.. todo:: + + Explain how to find out what blocks a given template provides. + Do you have to just look at the source? - If you want to add some code to a block but don't want to replace the entire - block, you can use Jinja's ``{{ super() }}`` tag:: - {% ckan_extends %} +Extending parent blocks with Jinja's ``{{ super() }}`` +------------------------------------------------------ - {% block secondary_content %} - {{ super() }} - Hello block world! - {% endblock %} +If you want to add some code to a block but don't want to replace the entire +block, you can use Jinja's ``{{ super() }}`` tag: - When the child block above is rendered, Jinja will replace the - ``{{ super() }}`` tag with the contents of the parent block. +.. literalinclude:: ../ckanext/example_theme/v5/templates/home/index.html - .. todo:: Make the ``super()`` example above into a proper included example. +When the child block above is rendered, Jinja will replace the +``{{ super() }}`` tag with the contents of the parent block. +The ``{{ super() }}`` tag can be placed anywhere in the block. -.. todo:: Need something here about what variables are available to templates: - c, h, g. etc. plus anything explicitly passed in by the controller or parent - template. Template helper functions ------------------------- @@ -261,13 +303,12 @@ One way for templates to get content out of CKAN is by calling CKAN's For example, let's replace the featured groups on the front page with an activity stream of the site's recently created, updated and deleted datasets. -Change the code in ``index.html`` to this: +Change the code in |index.html| to this: .. literalinclude:: ../ckanext/example_theme/v6/templates/home/index.html -Reload the CKAN front page in your browser (it shouldn't be necessary to -restart the web server, if you've only made changes to template files) and -you should see a new activity stream on the front page. +Reload the `CKAN front page`_ in your browser and you should see a new activity +stream. To call a template helper function we use a Jinja2 *expression* (code wrapped in ``{{ ... }}`` brackets), and we use the global variable ``h`` (available @@ -293,32 +334,51 @@ Let's add another item to our custom front page: a "dataset of the day". We'll add a custom template helper function to select the dataset to be shown. First, in our ``plugin.py`` file we need to implement :py:class:`~ckan.plugins.interfaces.ITemplateHelpers` and provide our helper -function. Change the contents of -``|extension_dir|/ckanext/example_theme/plugin.py`` to look like this: +function. Change the contents of ``plugin.py`` to look like this: + +.. todo: This probably breaks when the site has no datasets. .. literalinclude:: ../ckanext/example_theme/v7/plugin.py -.. todo:: Explain exactly what the new lines above do. +We've added a number of new features to ``plugin.py``. First, we defined a +function to get the dataset of the day from CKAN: - Mention why the helper is named as it is. +.. literalinclude:: ../ckanext/example_theme/v7/plugin.py + :pyobject: dataset_of_the_day + +This function uses CKAN's *action functions* to get the dataset from CKAN. +See :doc:`writing-extensions` for more about action functions. + +Next, we called :py:func:`~ckan.plugins.implements` to declare that your class +now implements :py:class:`~ckan.plugins.interfaces.ITemplateHelpers`: + +.. literalinclude:: ../ckanext/example_theme/v7/plugin.py + :start-after: # Declare that this plugin will implement ITemplateHelpers. + :end-before: def update_config(self, config): + +Finally, we implemented the +:py:meth:`~ckan.plugins.interfaces.ITemplateHelpers.get_helpers` method from +:py:class:`~ckan.plugins.interfaces.ITemplateHelpers` to register our function +as a template helper: + +.. literalinclude:: ../ckanext/example_theme/v7/plugin.py + :pyobject: ExampleThemePlugin.get_helpers Now that we've registered our helper function, we need to call it from our template. As with CKAN's default template helpers, templates access custom helpers via the global variable ``h``. -Edit ``|extension_dir|/ckanext/example_theme/home/index.html`` to look -like this: +Edit |index.html| to look like this: .. literalinclude:: ../ckanext/example_theme/v7/templates/home/index.html -Now restart your web server and reload your CKAN front page in your browser. -You should see the name of a random dataset appear on the page, and each time -you reload the page you'll get a different name. +Now reload your `CKAN front page`_ in your browser. You should see the title of +a random dataset appear on the page, and each time you reload the page you'll +get a different name. -Simply displaying the name of a dataset isn't very good. We want to show the -dataset's title not its name, have the title be hyperlinked to the dataset's -page, and also show some other information about the dataset such as its notes -and file formats. To display our dataset of the day nicely, we'll use CKAN's -*template snippets*. +Simply displaying the title of a dataset isn't very good. We want the dataset +to be hyperlinked to the it's page, and also to show some other information +about the dataset such as its notes and file formats. To display our dataset of +the day nicely, we'll use CKAN's *template snippets*. Template snippets @@ -343,7 +403,9 @@ a group's page, an organization's page or a user's page: .. literalinclude:: ../ckan/templates/snippets/package_item.html :end-before: #} -Let's change our ``index.html`` file to call this snippet: +.. todo:: Fix this docstring. + +Let's change our |index.html| file to call this snippet: .. literalinclude:: ../ckanext/example_theme/v8/templates/home/index.html @@ -353,8 +415,8 @@ the snippet as parameters. As in the ``package_item.html`` docstring above, each snippet's docstring should document the parameters it requires. In this example was pass just one parameter to the snippet: the dataset to be rendered. -If you reload your CKAN front page in your web browser now, you should see the -dataset of the day rendered nicely. +If you reload your `CKAN front page`_ in your web browser now, you should see +the dataset of the day rendered nicely. Adding your own template snippets @@ -366,17 +428,17 @@ their own snippets. To add template snippets, all a plugin needs to do is add a The snippets will be callable from other templates immediately. Let's add a custom snippet to change how our dataset of the day is displayed. -Create a new directory ``|extension_dir|/templates/snippets/`` containing -a file named ``example_theme_dataset_of_the_day.html`` with these contents: +Create a new directory |snippets_dir| containing a file named +``example_theme_dataset_of_the_day.html`` with these contents: .. literalinclude:: ../ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html -Now edit your ``index.html`` file and change to use our new snippet: +Now edit your |index.html| file and change to use our new snippet: .. literalinclude:: ../ckanext/example_theme/v9/templates/home/index.html -Restart your web server and reload your CKAN front page in your browser, and -you should see the display of the dataset of the day change. +Reload your `CKAN front page`_ in your browser, and you should see the display +of the dataset of the day change. .. todo:: @@ -412,6 +474,106 @@ you should see the display of the dataset of the day change. .. todo:: Verify whether this is completely true. +Global variables available to templates +--------------------------------------- + +.. _Pylons app_globals object: http://docs.pylonsproject.org/projects/pylons-webframework/en/latest/glossary.html#term-app-globals + +CKAN passes a number of variables that templates can use in Jinja statements or +expressions. For example, one such variable is ``app_globals``, the `Pylons +app_globals object`_, which can be used to access global attributes including +the config settings from the CKAN config file. + +.. todo:: Move these examples into a separate file. + +.. todo:: Change this example to use custom config settings, + dataset of the day, datasets of the day, show dataset of the day. + +:: + + {% ckan_extends %} + + {% block secondary_content %} + +

Site title: {{ g.site_title }}

+ + {% endblock %} + +``{{ g.site_title }}`` will output the :ref:`ckan.site_title` setting from your +config file (a string) into the web page. + +Some variables, such as :ref:`ckan.plugins`, are lists. We can use a +``{% for %}`` loop to print out the list of currently enabled plugins:: + +

Plugins enabled:

+
    + {% for plugin in g.plugins %} +
  • {{ plugin }}
    • + {% endfor %} +
+ +.. todo:: Mention what happens if you try to access a variable or attribute + that doesn't exist. + +.. todo:: Mention filters. And can ckan template helper functions be used as + filters? + +.. todo:: Mention tests. + +The following global variables are available to all templates: + +.. todo:: + + Add explanation of template variables and examples of how to use them in + Jinja2 templates. + + +``c`` + ``pylons.util.AttribSafeContextObj`` + + Using this is discouraged, better to use ``h``. + ``c`` is not available to snippets (but I think the rest are?) + +``h`` + ``ckan.config.environment._Helpers`` object + +``app_globals`` + ``ckan.lib.app_globals._Globals`` object + +``request`` + ``Request`` object + +``response`` + ``Response`` object + + .. todo:: Remove this? Doesn't appear to be used. + +``session`` + .. todo:: Remove this? Doesn't appear to be used. + +``N_`` + function ``gettext_noop`` + +``_`` + function ``ugettext`` + +``translator`` + ``gettext.NullTranslations`` instance + +``ungettext`` + function ``ungettext`` + +``actions`` + class ``ckan.model.authz.Action`` + + .. todo:: Remove this? Doesn't appear to be used and doesn't look like + something we want. + +.. todo:: + + Mention that any more variables explicitly passed in by the controller or + parent template are also available. + Bootstrap --------- @@ -466,3 +628,4 @@ Customizing CKAN's JavaScript * jQuery * Bootstrap's JavaScript stuff * Other stuff in javascript-module-tutorial.rst + From 51d2e7ae537ca92fec6d5f3dd68f5db1a20901fb Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Sep 2013 14:46:43 +0200 Subject: [PATCH 028/395] [#847] Add templating {{ super() }} example --- ckanext/example_theme/v5/templates/home/index.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ckanext/example_theme/v5/templates/home/index.html diff --git a/ckanext/example_theme/v5/templates/home/index.html b/ckanext/example_theme/v5/templates/home/index.html new file mode 100644 index 00000000000..d37f18e4957 --- /dev/null +++ b/ckanext/example_theme/v5/templates/home/index.html @@ -0,0 +1,14 @@ +{% ckan_extends %} + +{% block secondary_content %} + +

This paragraph will be added to the top of the + secondary_content block.

+ + {# Insert the contents of the original secondary_content block: #} + {{ super() }} + +

This paragraph will be added to the bottom of the + secondary_content block.

+ +{% endblock %} From 7dd3889a4e5a3262c0dd03b0c8a80b20221f7a2d Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Sep 2013 16:02:00 +0200 Subject: [PATCH 029/395] [#847] Rename all the theming example dirs This just makes it a little easier to keep track of which example is which --- .../{v1 => v1_empty_extension}/__init__.py | 0 .../{v1 => v1_empty_extension}/plugin.py | 0 .../{v2 => v2_empty_template}/__init__.py | 0 .../{v2 => v2_empty_template}/plugin.py | 0 .../templates/home/index.html | 0 ckanext/example_theme/v3/plugin.py | 1 - .../{v3 => v3_ckan_extends}/__init__.py | 0 .../example_theme/v3_ckan_extends/plugin.py | 1 + .../templates/home/index.html | 0 ckanext/example_theme/v4/plugin.py | 1 - .../{v4 => v4_block}/__init__.py | 0 ckanext/example_theme/v4_block/plugin.py | 1 + .../templates/home/index.html | 0 .../{v6 => v5_super}/__init__.py | 0 ckanext/example_theme/v5_super/plugin.py | 1 + .../templates/home/index.html | 0 ckanext/example_theme/v6/plugin.py | 1 - .../{v7 => v6_helper_function}/__init__.py | 0 .../v6_helper_function/plugin.py | 1 + .../templates/home/index.html | 0 .../__init__.py | 0 .../plugin.py | 0 .../templates/home/index.html | 0 ckanext/example_theme/v8/plugin.py | 1 - .../{v9 => v8_snippet}/__init__.py | 0 ckanext/example_theme/v8_snippet/plugin.py | 1 + .../templates/home/index.html | 0 ckanext/example_theme/v9/plugin.py | 1 - .../v9_custom_snippet/__init__.py | 0 .../example_theme/v9_custom_snippet/plugin.py | 1 + .../templates/home/index.html | 0 .../example_theme_dataset_of_the_day.html | 0 doc/theming.rst | 36 +++++++++---------- setup.py | 17 ++++----- 34 files changed, 33 insertions(+), 31 deletions(-) rename ckanext/example_theme/{v1 => v1_empty_extension}/__init__.py (100%) rename ckanext/example_theme/{v1 => v1_empty_extension}/plugin.py (100%) rename ckanext/example_theme/{v2 => v2_empty_template}/__init__.py (100%) rename ckanext/example_theme/{v2 => v2_empty_template}/plugin.py (100%) rename ckanext/example_theme/{v2 => v2_empty_template}/templates/home/index.html (100%) delete mode 120000 ckanext/example_theme/v3/plugin.py rename ckanext/example_theme/{v3 => v3_ckan_extends}/__init__.py (100%) create mode 120000 ckanext/example_theme/v3_ckan_extends/plugin.py rename ckanext/example_theme/{v3 => v3_ckan_extends}/templates/home/index.html (100%) delete mode 120000 ckanext/example_theme/v4/plugin.py rename ckanext/example_theme/{v4 => v4_block}/__init__.py (100%) create mode 120000 ckanext/example_theme/v4_block/plugin.py rename ckanext/example_theme/{v4 => v4_block}/templates/home/index.html (100%) rename ckanext/example_theme/{v6 => v5_super}/__init__.py (100%) create mode 120000 ckanext/example_theme/v5_super/plugin.py rename ckanext/example_theme/{v5 => v5_super}/templates/home/index.html (100%) delete mode 120000 ckanext/example_theme/v6/plugin.py rename ckanext/example_theme/{v7 => v6_helper_function}/__init__.py (100%) create mode 120000 ckanext/example_theme/v6_helper_function/plugin.py rename ckanext/example_theme/{v6 => v6_helper_function}/templates/home/index.html (100%) rename ckanext/example_theme/{v8 => v7_custom_helper_function}/__init__.py (100%) rename ckanext/example_theme/{v7 => v7_custom_helper_function}/plugin.py (100%) rename ckanext/example_theme/{v7 => v7_custom_helper_function}/templates/home/index.html (100%) delete mode 120000 ckanext/example_theme/v8/plugin.py rename ckanext/example_theme/{v9 => v8_snippet}/__init__.py (100%) create mode 120000 ckanext/example_theme/v8_snippet/plugin.py rename ckanext/example_theme/{v8 => v8_snippet}/templates/home/index.html (100%) delete mode 120000 ckanext/example_theme/v9/plugin.py create mode 100644 ckanext/example_theme/v9_custom_snippet/__init__.py create mode 120000 ckanext/example_theme/v9_custom_snippet/plugin.py rename ckanext/example_theme/{v9 => v9_custom_snippet}/templates/home/index.html (100%) rename ckanext/example_theme/{v9 => v9_custom_snippet}/templates/snippets/example_theme_dataset_of_the_day.html (100%) diff --git a/ckanext/example_theme/v1/__init__.py b/ckanext/example_theme/v1_empty_extension/__init__.py similarity index 100% rename from ckanext/example_theme/v1/__init__.py rename to ckanext/example_theme/v1_empty_extension/__init__.py diff --git a/ckanext/example_theme/v1/plugin.py b/ckanext/example_theme/v1_empty_extension/plugin.py similarity index 100% rename from ckanext/example_theme/v1/plugin.py rename to ckanext/example_theme/v1_empty_extension/plugin.py diff --git a/ckanext/example_theme/v2/__init__.py b/ckanext/example_theme/v2_empty_template/__init__.py similarity index 100% rename from ckanext/example_theme/v2/__init__.py rename to ckanext/example_theme/v2_empty_template/__init__.py diff --git a/ckanext/example_theme/v2/plugin.py b/ckanext/example_theme/v2_empty_template/plugin.py similarity index 100% rename from ckanext/example_theme/v2/plugin.py rename to ckanext/example_theme/v2_empty_template/plugin.py diff --git a/ckanext/example_theme/v2/templates/home/index.html b/ckanext/example_theme/v2_empty_template/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v2/templates/home/index.html rename to ckanext/example_theme/v2_empty_template/templates/home/index.html diff --git a/ckanext/example_theme/v3/plugin.py b/ckanext/example_theme/v3/plugin.py deleted file mode 120000 index e2729b2750a..00000000000 --- a/ckanext/example_theme/v3/plugin.py +++ /dev/null @@ -1 +0,0 @@ -../v2/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v3/__init__.py b/ckanext/example_theme/v3_ckan_extends/__init__.py similarity index 100% rename from ckanext/example_theme/v3/__init__.py rename to ckanext/example_theme/v3_ckan_extends/__init__.py diff --git a/ckanext/example_theme/v3_ckan_extends/plugin.py b/ckanext/example_theme/v3_ckan_extends/plugin.py new file mode 120000 index 00000000000..07b1f31cf30 --- /dev/null +++ b/ckanext/example_theme/v3_ckan_extends/plugin.py @@ -0,0 +1 @@ +../v2_empty_template/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v3/templates/home/index.html b/ckanext/example_theme/v3_ckan_extends/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v3/templates/home/index.html rename to ckanext/example_theme/v3_ckan_extends/templates/home/index.html diff --git a/ckanext/example_theme/v4/plugin.py b/ckanext/example_theme/v4/plugin.py deleted file mode 120000 index e2729b2750a..00000000000 --- a/ckanext/example_theme/v4/plugin.py +++ /dev/null @@ -1 +0,0 @@ -../v2/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v4/__init__.py b/ckanext/example_theme/v4_block/__init__.py similarity index 100% rename from ckanext/example_theme/v4/__init__.py rename to ckanext/example_theme/v4_block/__init__.py diff --git a/ckanext/example_theme/v4_block/plugin.py b/ckanext/example_theme/v4_block/plugin.py new file mode 120000 index 00000000000..b33c6582919 --- /dev/null +++ b/ckanext/example_theme/v4_block/plugin.py @@ -0,0 +1 @@ +../v3_ckan_extends/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v4/templates/home/index.html b/ckanext/example_theme/v4_block/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v4/templates/home/index.html rename to ckanext/example_theme/v4_block/templates/home/index.html diff --git a/ckanext/example_theme/v6/__init__.py b/ckanext/example_theme/v5_super/__init__.py similarity index 100% rename from ckanext/example_theme/v6/__init__.py rename to ckanext/example_theme/v5_super/__init__.py diff --git a/ckanext/example_theme/v5_super/plugin.py b/ckanext/example_theme/v5_super/plugin.py new file mode 120000 index 00000000000..3848f969be3 --- /dev/null +++ b/ckanext/example_theme/v5_super/plugin.py @@ -0,0 +1 @@ +../v4_block/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v5/templates/home/index.html b/ckanext/example_theme/v5_super/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v5/templates/home/index.html rename to ckanext/example_theme/v5_super/templates/home/index.html diff --git a/ckanext/example_theme/v6/plugin.py b/ckanext/example_theme/v6/plugin.py deleted file mode 120000 index e2729b2750a..00000000000 --- a/ckanext/example_theme/v6/plugin.py +++ /dev/null @@ -1 +0,0 @@ -../v2/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v7/__init__.py b/ckanext/example_theme/v6_helper_function/__init__.py similarity index 100% rename from ckanext/example_theme/v7/__init__.py rename to ckanext/example_theme/v6_helper_function/__init__.py diff --git a/ckanext/example_theme/v6_helper_function/plugin.py b/ckanext/example_theme/v6_helper_function/plugin.py new file mode 120000 index 00000000000..ea0cc975fd0 --- /dev/null +++ b/ckanext/example_theme/v6_helper_function/plugin.py @@ -0,0 +1 @@ +../v5_super/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v6/templates/home/index.html b/ckanext/example_theme/v6_helper_function/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v6/templates/home/index.html rename to ckanext/example_theme/v6_helper_function/templates/home/index.html diff --git a/ckanext/example_theme/v8/__init__.py b/ckanext/example_theme/v7_custom_helper_function/__init__.py similarity index 100% rename from ckanext/example_theme/v8/__init__.py rename to ckanext/example_theme/v7_custom_helper_function/__init__.py diff --git a/ckanext/example_theme/v7/plugin.py b/ckanext/example_theme/v7_custom_helper_function/plugin.py similarity index 100% rename from ckanext/example_theme/v7/plugin.py rename to ckanext/example_theme/v7_custom_helper_function/plugin.py diff --git a/ckanext/example_theme/v7/templates/home/index.html b/ckanext/example_theme/v7_custom_helper_function/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v7/templates/home/index.html rename to ckanext/example_theme/v7_custom_helper_function/templates/home/index.html diff --git a/ckanext/example_theme/v8/plugin.py b/ckanext/example_theme/v8/plugin.py deleted file mode 120000 index 6f55fa44e01..00000000000 --- a/ckanext/example_theme/v8/plugin.py +++ /dev/null @@ -1 +0,0 @@ -../v7/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v9/__init__.py b/ckanext/example_theme/v8_snippet/__init__.py similarity index 100% rename from ckanext/example_theme/v9/__init__.py rename to ckanext/example_theme/v8_snippet/__init__.py diff --git a/ckanext/example_theme/v8_snippet/plugin.py b/ckanext/example_theme/v8_snippet/plugin.py new file mode 120000 index 00000000000..162cf013c01 --- /dev/null +++ b/ckanext/example_theme/v8_snippet/plugin.py @@ -0,0 +1 @@ +../v7_custom_helper_function/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v8/templates/home/index.html b/ckanext/example_theme/v8_snippet/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v8/templates/home/index.html rename to ckanext/example_theme/v8_snippet/templates/home/index.html diff --git a/ckanext/example_theme/v9/plugin.py b/ckanext/example_theme/v9/plugin.py deleted file mode 120000 index 74b99bdf60c..00000000000 --- a/ckanext/example_theme/v9/plugin.py +++ /dev/null @@ -1 +0,0 @@ -../v8/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v9_custom_snippet/__init__.py b/ckanext/example_theme/v9_custom_snippet/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckanext/example_theme/v9_custom_snippet/plugin.py b/ckanext/example_theme/v9_custom_snippet/plugin.py new file mode 120000 index 00000000000..4994c490fd3 --- /dev/null +++ b/ckanext/example_theme/v9_custom_snippet/plugin.py @@ -0,0 +1 @@ +../v8_snippet/plugin.py \ No newline at end of file diff --git a/ckanext/example_theme/v9/templates/home/index.html b/ckanext/example_theme/v9_custom_snippet/templates/home/index.html similarity index 100% rename from ckanext/example_theme/v9/templates/home/index.html rename to ckanext/example_theme/v9_custom_snippet/templates/home/index.html diff --git a/ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html b/ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html similarity index 100% rename from ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html rename to ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html diff --git a/doc/theming.rst b/doc/theming.rst index 494d5320733..d611191656d 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -59,7 +59,7 @@ an extension and plugin. For a detailed explanation of the steps below, see 2. Create the file |plugin.py| with the following contents: - .. literalinclude:: ../ckanext/example_theme/v1/plugin.py + .. literalinclude:: ../ckanext/example_theme/v1_empty_extension/plugin.py 3. Edit the ``entry_points`` in |setup.py|:: @@ -114,13 +114,13 @@ directory containing templates file that override the default ones. Edit the |plugin.py| file that we created earlier, so that it looks like this: -.. literalinclude:: ../ckanext/example_theme/v2/plugin.py +.. literalinclude:: ../ckanext/example_theme/v2_empty_template/plugin.py This new code does a few things: 1. It imports CKAN's *plugins toolkit* module: - .. literalinclude:: ../ckanext/example_theme/v2/plugin.py + .. literalinclude:: ../ckanext/example_theme/v2_empty_template/plugin.py :start-after: import ckan.plugins as plugins :end-before: class ExampleThemePlugin(plugins.SingletonPlugin): @@ -132,7 +132,7 @@ This new code does a few things: implements the :py:class:`~ckan.plugins.interfaces.IConfigurer` plugin interface: - .. literalinclude:: ../ckanext/example_theme/v2/plugin.py + .. literalinclude:: ../ckanext/example_theme/v2_empty_template/plugin.py :start-after: # Declare that this class implements IConfigurer. :end-before: def update_config( @@ -148,7 +148,7 @@ This new code does a few things: is the only method declated in the :py:class:`~ckan.plugins.interfaces.IConfigurer` interface: - .. literalinclude:: ../ckanext/example_theme/v2/plugin.py + .. literalinclude:: ../ckanext/example_theme/v2_empty_template/plugin.py :pyobject: ExampleThemePlugin.update_config CKAN will call this method when it starts up, to give our plugin a chance to @@ -229,7 +229,7 @@ declare that our ``home/index.html`` template extends the default ``home/index.html`` template, instead of completely replacing it. Edit the empty ``index.html`` file you just created, and add one line: -.. literalinclude:: ../ckanext/example_theme/v3/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v3_ckan_extends/templates/home/index.html If you now reload the `CKAN front page`_ in your browser, you should see the normal front page appear again. When CKAN processes our ``index.html`` file, @@ -269,7 +269,7 @@ When a custom template file extends one of CKAN's default template files using template with its own code by using ``{% block %}``. Edit your ``index.html`` file again and change the contents to: -.. literalinclude:: ../ckanext/example_theme/v4/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v4_block/templates/home/index.html Reload the `CKAN front page`_ in your browser. You should see that the featured groups section of the page has been replaced, @@ -287,7 +287,7 @@ Extending parent blocks with Jinja's ``{{ super() }}`` If you want to add some code to a block but don't want to replace the entire block, you can use Jinja's ``{{ super() }}`` tag: -.. literalinclude:: ../ckanext/example_theme/v5/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v5_super/templates/home/index.html When the child block above is rendered, Jinja will replace the ``{{ super() }}`` tag with the contents of the parent block. @@ -305,7 +305,7 @@ For example, let's replace the featured groups on the front page with an activity stream of the site's recently created, updated and deleted datasets. Change the code in |index.html| to this: -.. literalinclude:: ../ckanext/example_theme/v6/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v6_helper_function/templates/home/index.html Reload the `CKAN front page`_ in your browser and you should see a new activity stream. @@ -314,7 +314,7 @@ To call a template helper function we use a Jinja2 *expression* (code wrapped in ``{{ ... }}`` brackets), and we use the global variable ``h`` (available to all templates) to access the helper: -.. literalinclude:: ../ckanext/example_theme/v6/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v6_helper_function/templates/home/index.html :start-after: {% block secondary_content %} :end-before: {% endblock %} @@ -338,12 +338,12 @@ function. Change the contents of ``plugin.py`` to look like this: .. todo: This probably breaks when the site has no datasets. -.. literalinclude:: ../ckanext/example_theme/v7/plugin.py +.. literalinclude:: ../ckanext/example_theme/v7_custom_helper_function/plugin.py We've added a number of new features to ``plugin.py``. First, we defined a function to get the dataset of the day from CKAN: -.. literalinclude:: ../ckanext/example_theme/v7/plugin.py +.. literalinclude:: ../ckanext/example_theme/v7_custom_helper_function/plugin.py :pyobject: dataset_of_the_day This function uses CKAN's *action functions* to get the dataset from CKAN. @@ -352,7 +352,7 @@ See :doc:`writing-extensions` for more about action functions. Next, we called :py:func:`~ckan.plugins.implements` to declare that your class now implements :py:class:`~ckan.plugins.interfaces.ITemplateHelpers`: -.. literalinclude:: ../ckanext/example_theme/v7/plugin.py +.. literalinclude:: ../ckanext/example_theme/v7_custom_helper_function/plugin.py :start-after: # Declare that this plugin will implement ITemplateHelpers. :end-before: def update_config(self, config): @@ -361,7 +361,7 @@ Finally, we implemented the :py:class:`~ckan.plugins.interfaces.ITemplateHelpers` to register our function as a template helper: -.. literalinclude:: ../ckanext/example_theme/v7/plugin.py +.. literalinclude:: ../ckanext/example_theme/v7_custom_helper_function/plugin.py :pyobject: ExampleThemePlugin.get_helpers Now that we've registered our helper function, we need to call it from our @@ -369,7 +369,7 @@ template. As with CKAN's default template helpers, templates access custom helpers via the global variable ``h``. Edit |index.html| to look like this: -.. literalinclude:: ../ckanext/example_theme/v7/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v7_custom_helper_function/templates/home/index.html Now reload your `CKAN front page`_ in your browser. You should see the title of a random dataset appear on the page, and each time you reload the page you'll @@ -407,7 +407,7 @@ a group's page, an organization's page or a user's page: Let's change our |index.html| file to call this snippet: -.. literalinclude:: ../ckanext/example_theme/v8/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v8_snippet/templates/home/index.html The ``{% snippet %}`` tag takes one or more arguments. The first argument is the name of the snippet to call. Any further arguments will be passed into @@ -431,11 +431,11 @@ Let's add a custom snippet to change how our dataset of the day is displayed. Create a new directory |snippets_dir| containing a file named ``example_theme_dataset_of_the_day.html`` with these contents: -.. literalinclude:: ../ckanext/example_theme/v9/templates/snippets/example_theme_dataset_of_the_day.html +.. literalinclude:: ../ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html Now edit your |index.html| file and change to use our new snippet: -.. literalinclude:: ../ckanext/example_theme/v9/templates/home/index.html +.. literalinclude:: ../ckanext/example_theme/v9_custom_snippet/templates/home/index.html Reload your `CKAN front page`_ in your browser, and you should see the display of the dataset of the day change. diff --git a/setup.py b/setup.py index 832c203761b..a5c7b98c9cd 100644 --- a/setup.py +++ b/setup.py @@ -122,14 +122,15 @@ recline_preview=ckanext.reclinepreview.plugin:ReclinePreview example_itemplatehelpers=ckanext.example_itemplatehelpers.plugin:ExampleITemplateHelpersPlugin example_idatasetform=ckanext.example_idatasetform.plugin:ExampleIDatasetFormPlugin - example_theme_v1=ckanext.example_theme.v1.plugin:ExampleThemePlugin - example_theme_v2=ckanext.example_theme.v2.plugin:ExampleThemePlugin - example_theme_v3=ckanext.example_theme.v3.plugin:ExampleThemePlugin - example_theme_v4=ckanext.example_theme.v4.plugin:ExampleThemePlugin - example_theme_v6=ckanext.example_theme.v6.plugin:ExampleThemePlugin - example_theme_v7=ckanext.example_theme.v7.plugin:ExampleThemePlugin - example_theme_v8=ckanext.example_theme.v8.plugin:ExampleThemePlugin - example_theme_v9=ckanext.example_theme.v9.plugin:ExampleThemePlugin + example_theme_v1=ckanext.example_theme.v1_empty_extension.plugin:ExampleThemePlugin + example_theme_v2=ckanext.example_theme.v2_empty_template.plugin:ExampleThemePlugin + example_theme_v3=ckanext.example_theme.v3_ckan_extends.plugin:ExampleThemePlugin + example_theme_v4=ckanext.example_theme.v4_block.plugin:ExampleThemePlugin + example_theme_v5=ckanext.example_theme.v5_super.plugin:ExampleThemePlugin + example_theme_v6=ckanext.example_theme.v6_helper_function.plugin:ExampleThemePlugin + example_theme_v7=ckanext.example_theme.v7_custom_helper_function.plugin:ExampleThemePlugin + example_theme_v8=ckanext.example_theme.v8_snippet.plugin:ExampleThemePlugin + example_theme_v9=ckanext.example_theme.v9_custom_snippet.plugin:ExampleThemePlugin [ckan.system_plugins] domain_object_mods = ckan.model.modification:DomainObjectModificationExtension From 8a195e9eb9e2a8baaf6050f3e48c0fb4077c0632 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Sep 2013 16:02:38 +0200 Subject: [PATCH 030/395] [#847] Add missing templates_dir substitution --- doc/theming.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/theming.rst b/doc/theming.rst index d611191656d..1f4d8389efb 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -4,6 +4,7 @@ .. |extension_dir| replace:: ``ckanext-example_theme`` .. |setup.py| replace:: ``ckanext-example_theme/setup.py`` ``ckan.plugins`` .. |plugin.py| replace:: ``ckanext-example_theme/ckanext/example_theme/plugin.py`` +.. |templates_dir| replace:: ``ckanext-example_theme/ckanext/example_theme/templates`` .. |index.html| replace:: ``ckanext-example_theme/ckanext/example_theme/templates/home/index.html`` .. |snippets_dir| replace:: ``ckanext-example_theme/ckanext/example_theme/templates/snippets`` From 655d56fbcfc19abd0a343ed590506ec50a191ab0 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 13:25:39 +0200 Subject: [PATCH 031/395] [#847] Tweak a theming example --- ckanext/example_theme/v8_snippet/templates/home/index.html | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/example_theme/v8_snippet/templates/home/index.html b/ckanext/example_theme/v8_snippet/templates/home/index.html index 6cbf40da9a2..a94d4a20682 100644 --- a/ckanext/example_theme/v8_snippet/templates/home/index.html +++ b/ckanext/example_theme/v8_snippet/templates/home/index.html @@ -4,6 +4,7 @@ {{ h.recently_changed_packages_activity_stream() }} +

Dataset of the day

{% snippet 'snippets/package_item.html', package=h.example_theme_dataset_of_the_day() %} From d6abcde76efde9659f88a518ebe5b8db9a0eefe5 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 13:29:23 +0200 Subject: [PATCH 032/395] [#847] Improve custom snippet example --- .../templates/home/index.html | 3 +- .../example_theme_dataset_of_the_day.html | 30 ++++++++++--------- doc/theming.rst | 24 ++++++++++----- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/ckanext/example_theme/v9_custom_snippet/templates/home/index.html b/ckanext/example_theme/v9_custom_snippet/templates/home/index.html index 2cf4f00533d..42a2b7aec63 100644 --- a/ckanext/example_theme/v9_custom_snippet/templates/home/index.html +++ b/ckanext/example_theme/v9_custom_snippet/templates/home/index.html @@ -4,7 +4,6 @@ {{ h.recently_changed_packages_activity_stream() }} - {% snippet 'snippets/example_theme_dataset_of_the_day.html', - dataset=h.example_theme_dataset_of_the_day() %} + {% snippet 'snippets/example_theme_dataset_of_the_day.html' %} {% endblock %} diff --git a/ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html b/ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html index 102a01e18b9..2f0d78c9b54 100644 --- a/ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html +++ b/ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html @@ -1,15 +1,17 @@ -{# Renders a preview of the dataset of the day. - -dataset - The dataset to preview - -#} -{% set notes = h.markdown_extract(dataset.notes, extract_length=truncate) %} -
-

- {{ h.link_to(h.truncate(title, truncate_title), - h.url_for(controller='package', action='read', id=dataset.name)) }} -

- {% if notes %} -
{{ notes|urlize }}
- {% endif %} +{# Renders a preview of the dataset of the day. #} + +
+ +
+

Dataset of the day

+
+ +
    + {# Call the core package_item.html snippet to render the dataset. #} + {% snippet 'snippets/package_item.html', + package=h.example_theme_dataset_of_the_day(), + item_class='module-content' + %} +
+
diff --git a/doc/theming.rst b/doc/theming.rst index e901bf5ed78..f0ec89d328b 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -428,22 +428,32 @@ their own snippets. To add template snippets, all a plugin needs to do is add a ``snippets`` directory in its ``templates`` directory, and start adding files. The snippets will be callable from other templates immediately. -Let's add a custom snippet to change how our dataset of the day is displayed. +Let's create a custom snippet to display our dataset of the day, and put the +``

Dataset of the day

`` heading and the code to call the helper +function to retrieve the dataset into the snippet, so that we can reuse the +whole thing on different parts of the site if we want to. + Create a new directory |snippets_dir| containing a file named ``example_theme_dataset_of_the_day.html`` with these contents: .. literalinclude:: ../ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html -Now edit your |index.html| file and change to use our new snippet: - -.. literalinclude:: ../ckanext/example_theme/v9_custom_snippet/templates/home/index.html +As you can see, snippets can call other snippets - our custom snippet actually +calls the default snippet to render the dataset itself: -Reload your `CKAN front page`_ in your browser, and you should see the display -of the dataset of the day change. +.. literalinclude:: ../ckanext/example_theme/v9_custom_snippet/templates/snippets/example_theme_dataset_of_the_day.html + :start-after: {# Call the core package_item.html snippet to render the dataset. #} + :end-before: %} .. todo:: - Make the snippet better, and explain the snippet line-by-line. + Explain the HTML and CSS being used in the snippet above, and where it comes + from. + +Now edit your |index.html| file and change it to use our new snippet instead of +the default one: + +.. literalinclude:: ../ckanext/example_theme/v9_custom_snippet/templates/home/index.html .. warning:: Snippet overriding From 7de4aac693b77bb89e8b0a4590858cb1a40c5609 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 13:30:00 +0200 Subject: [PATCH 033/395] [#847] Correct a couple of typos --- doc/theming.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/theming.rst b/doc/theming.rst index f0ec89d328b..059b3d741e5 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -146,7 +146,7 @@ This new code does a few things: 3. It implements the :py:meth:`~ckan.plugins.interfaces.IConfigurer.update_config` method, which - is the only method declated in the + is the only method declared in the :py:class:`~ckan.plugins.interfaces.IConfigurer` interface: .. literalinclude:: ../ckanext/example_theme/v2_empty_template/plugin.py @@ -377,7 +377,7 @@ a random dataset appear on the page, and each time you reload the page you'll get a different name. Simply displaying the title of a dataset isn't very good. We want the dataset -to be hyperlinked to the it's page, and also to show some other information +to be hyperlinked to its page, and also to show some other information about the dataset such as its notes and file formats. To display our dataset of the day nicely, we'll use CKAN's *template snippets*. From a56c231fef21f6aee297af356da378ace568f2fc Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 13:30:31 +0200 Subject: [PATCH 034/395] [#847] Tweak warning about snippet overriding --- doc/theming.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/theming.rst b/doc/theming.rst index 059b3d741e5..6d3ca1c615e 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -455,8 +455,9 @@ the default one: .. literalinclude:: ../ckanext/example_theme/v9_custom_snippet/templates/home/index.html -.. warning:: Snippet overriding +.. warning:: + Default snippets can be overridden. If a plugin adds a snippet with the same name as one of CKAN's default snippets, the plugin's snippet will override the default snippet wherever the default snippet is used. From f8fe2d84c28776c61f0d9b15bd71f259f03258dc Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 13:33:58 +0200 Subject: [PATCH 035/395] [#847] Add note about snippet filenames --- doc/theming.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/theming.rst b/doc/theming.rst index 6d3ca1c615e..e3095ea2202 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -465,6 +465,10 @@ the default one: Also if two plugins both have snippets with the same name, one of the snippets will override the other. <-- TODO: Verify whether this is true + To avoid unintended conflicts, we recommend that snippet filenames begin + with the name of the extension they belong to, e.g. + ``snippets/example_theme_*.html``. + .. todo:: Exactly what order are ``snippets`` directories read in, and what From 30653190c8146cd995385d5a55517589de95f324 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 13:34:41 +0200 Subject: [PATCH 036/395] [#847] Correct note about snippets and global variables --- doc/theming.rst | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/doc/theming.rst b/doc/theming.rst index e3095ea2202..5adf184a024 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -476,19 +476,14 @@ the default one: .. note:: - Snippets don't have access to the global template context, so global - variables such as ``c``, ``h`` and ``g`` that are available to templates - are not available to snippets. + Snippets don't have access to the global template context variable, + ``c`` (see :ref:`global variables`). Snippets *can* access other + global variables such as ``h``, ``app_globals`` and ``request``, as well as + any variables explicitly passed into the snippet by the parent template when + it calls the snippet with a ``{% snippet %}`` tag. - The only variables available to snippets are those explicitly passed into - the snippet by the parent template, when it calls the snippet using a - ``{% snippet %}`` tag. - - Keeping snippets "modular" in this way makes debugging template problems - much easier. - - .. todo:: Verify whether this is completely true. +.. _global variables: Global variables available to templates --------------------------------------- From 40ef2f95a7cbd5c48a88d8446490f1a4c8bbd12c Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 13:43:05 +0200 Subject: [PATCH 037/395] [#847] Add a note to the theming docs --- doc/theming.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/theming.rst b/doc/theming.rst index 5adf184a024..f639e097f1e 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -107,6 +107,8 @@ datasets page at ``/dataset`` is generated from ``ckan/templates/package/search.html``, etc. +.. _template overriding: + Replacing a default template file --------------------------------- @@ -428,6 +430,12 @@ their own snippets. To add template snippets, all a plugin needs to do is add a ``snippets`` directory in its ``templates`` directory, and start adding files. The snippets will be callable from other templates immediately. +.. note:: + + For CKAN to find your plugins' snippets directories, you should already have + added your plugin's custom template directory to CKAN, see :ref:`template + overriding`. + Let's create a custom snippet to display our dataset of the day, and put the ``

Dataset of the day

`` heading and the code to call the helper function to retrieve the dataset into the snippet, so that we can reuse the From 0af3d0df4df9d24b61be5d83adf204911bf34732 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Sep 2013 19:46:10 +0200 Subject: [PATCH 038/395] [#847] Add basic intro to Jinja Expressions, tags, and what's available in the global namespace. --- doc/theming.rst | 118 +++++++++++++++++++++++- doc/theming/variables-and-functions.rst | 71 ++++++++++++++ 2 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 doc/theming/variables-and-functions.rst diff --git a/doc/theming.rst b/doc/theming.rst index f639e097f1e..ca29ce9c715 100644 --- a/doc/theming.rst +++ b/doc/theming.rst @@ -213,15 +213,121 @@ an empty file. Jinja2 ------ -.. todo:: Brief introduction to Jinja2 here. - CKAN template files are written in the `Jinja2`_ templating language. Jinja template files, such as our ``index.html`` file, are simply text files that, when processed, generate any text-based output format such as ``HTML``, ``XML``, ``CSV``, etc. Most of the templates file in CKAN generate ``HTML``. -In Jinja templates snippets of text like ``{% ... %}`` are Jinja *tags* that -control the logic of the template. +We'll introduce some Jinja2 basics below. Jinja2 templates have many more +features than these, for full details see the +`Jinja2 docs `_. + + +Expressions and variables +````````````````````````` + +Jinja2 *expressions* are snippets of code between ``{{ ... }}`` delimiters, +when a template is rendered any expressions are evaluated and replaced with +the resulting value. + +The simplest use of an expression is to display the value of a variable, for +example ``{{ foo }}`` in a template file will be replaced with the value of the +variable ``foo`` when the template is rendered. + +.. _Pylons app_globals object: http://docs.pylonsproject.org/projects/pylons-webframework/en/latest/glossary.html#term-app-globals + +CKAN makes a number of global variables available to all templates. One such +variable is ``app_globals``, the `Pylons app_globals object`_, which can be +used to access certain global attributes including some of the settings from +your CKAN config file. For example, to display the value of the +:ref:`ckan.site_title` setting from your config file you would put this code in +any template file: + +.. todo:: Move these examples into separate files. + +.. todo:: Should we be using ``app_globals`` or ``g``? + +:: + +

The title of this site is: {{ app_globals.site_title }}.

+ +.. note:: + + Not all config settings are available to templates via ``app_globals``. + The :ref:`sqlalchemy.url` setting, for example, contains your database + password, so making that variable available to templates might be a security + risk. + + If you've added your own custom options to your config file, these will not + be available in ``app_globals``. + + .. todo:: Insert cross-ref to custom config options section. + +.. note:: + + Jinja2 expressions can do much more than print out the values of variables, + for example + they can call Jinja2's `global functions `_, + CKAN's :ref:`template helper functions