From 0b30897cd28bbc62efa2d9811cb3bff8af6627ae Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Wed, 2 May 2012 12:42:33 +0100 Subject: [PATCH 001/278] [2347] Implemented related dashboard with filter on type and tidied up some of the related templates --- ckan/config/routing.py | 1 + ckan/controllers/related.py | 50 ++++++++++++++++- ckan/logic/action/get.py | 23 +++++--- ckan/templates/_util.html | 4 +- .../{_snippet => related}/add-related.html | 0 ckan/templates/related/dashboard.html | 56 +++++++++++++++++++ .../{package => related}/related_list.html | 6 +- 7 files changed, 125 insertions(+), 15 deletions(-) rename ckan/templates/{_snippet => related}/add-related.html (100%) create mode 100644 ckan/templates/related/dashboard.html rename ckan/templates/{package => related}/related_list.html (95%) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 43fa8fd9e1f..1ad271e9c76 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -155,6 +155,7 @@ def make_map(): with SubMapper(map, controller='related') as m: m.connect('related_list', '/dataset/{id}/related', action='list') m.connect('related_read', '/dataset/{id}/related/{related_id}', action='read') + m.connect('related_dashboard', '/apps', action='dashboard') with SubMapper(map, controller='package') as m: m.connect('/dataset', action='search') diff --git a/ckan/controllers/related.py b/ckan/controllers/related.py index 85042aaa8eb..fb6effdd5ee 100644 --- a/ckan/controllers/related.py +++ b/ckan/controllers/related.py @@ -4,13 +4,59 @@ import ckan.logic as logic import ckan.lib.base as base import ckan.lib.helpers as h +import urllib c = base.c class RelatedController(base.BaseController): - def list(self, id): + def dashboard(self): + """ List all related items regardless of dataset """ + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author, 'extras_as_string': True, + 'for_view': True} + data_dict = { + 'type_filter': base.request.params.get('type', '') + } + + params_nopage = [(k, v) for k,v in base.request.params.items() + if k != 'page'] + try: + page = int(base.request.params.get('page', 1)) + except ValueError, e: + abort(400, ('"page" parameter must be an integer')) + + # Update ordering in the context + query = logic.get_action('related_list')(context,data_dict) + + def search_url(params): + url = h.url_for(controller='related', action='dashboard') + params = [(k, v.encode('utf-8') + if isinstance(v, basestring) else str(v)) + for k, v in params] + return url + u'?' + urllib.urlencode(params) + def pager_url(q=None, page=None): + params = list(params_nopage) + params.append(('page', page)) + return search_url(params) + + + c.page = h.Page( + collection=query.all(), + page=page, + url=pager_url, + item_count=query.count(), + items_per_page=8 + ) + + c.filters = dict(params_nopage) + + return base.render( "related/dashboard.html") + + + def list(self, id): + """ List all related items for a specific dataset """ context = {'model': model, 'session': model.Session, 'user': c.user or c.author, 'extras_as_string': True, 'for_view': True} @@ -32,5 +78,5 @@ def list(self, id): c.related_count = len(c.pkg.related) - return base.render( "package/related_list.html") + return base.render( "related/related_list.html") diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index b2afe947171..b0e315e789d 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -138,8 +138,8 @@ def related_show(context, data_dict=None): def related_list(context, data_dict=None): """ - List the related items for a specific package which should be - mentioned in the data_dict + List the related items which may be for a specific package which + should be mentioned in the data_dict context: model - The CKAN model module @@ -150,6 +150,9 @@ def related_list(context, data_dict=None): id - The ID of the dataset to which we want to list related items or dataset - The dataset (package) model + + If neither value is in the data_dict then all related items will + be returned, and the ordering requested will be applied. """ model = context['model'] session = context['session'] @@ -158,14 +161,18 @@ def related_list(context, data_dict=None): if not dataset: dataset = model.Package.get(data_dict.get('id')) - if not dataset: - raise NotFound - check_access('related_show',context, data_dict) - relateds = model.Related.get_for_dataset(dataset, status='active') - related_items = (r.related for r in relateds) - related_list = model_dictize.related_list_dictize( related_items, context) + related_list = [] + if not dataset: + related_list = model.Session.query(model.Related) + tfilter = data_dict.get('type_filter', None) + if tfilter: + related_list = related_list.filter(model.Related.type == tfilter) + else: + relateds = model.Related.get_for_dataset(dataset, status='active') + related_items = (r.related for r in relateds) + related_list = model_dictize.related_list_dictize( related_items, context) return related_list diff --git a/ckan/templates/_util.html b/ckan/templates/_util.html index 5aa7c077c7e..73f976ffea1 100644 --- a/ckan/templates/_util.html +++ b/ckan/templates/_util.html @@ -125,10 +125,10 @@ - +
  • - + diff --git a/ckan/templates/_snippet/add-related.html b/ckan/templates/related/add-related.html similarity index 100% rename from ckan/templates/_snippet/add-related.html rename to ckan/templates/related/add-related.html diff --git a/ckan/templates/related/dashboard.html b/ckan/templates/related/dashboard.html new file mode 100644 index 00000000000..fc519b7baec --- /dev/null +++ b/ckan/templates/related/dashboard.html @@ -0,0 +1,56 @@ + + + no-sidebar + + + Apps + + + Applications + + +
    +

    Showing items ${c.page.first_item} - ${c.page.last_item} of ${c.page.item_count} related items found

    +

    ${c.page.item_count} related items found

    + +
    +
    + + + + + + +
    +
    + +
      + + ${related_summary(related,False)} + +
    + + ${c.page.pager()} + + +
    + + + + diff --git a/ckan/templates/package/related_list.html b/ckan/templates/related/related_list.html similarity index 95% rename from ckan/templates/package/related_list.html rename to ckan/templates/related/related_list.html index baced803519..324c3817e82 100644 --- a/ckan/templates/package/related_list.html +++ b/ckan/templates/related/related_list.html @@ -12,7 +12,7 @@ py:strip=""> no-sidebar - + ${c.pkg_dict.get('title', c.pkg_dict['name'])} - Related @@ -31,7 +31,7 @@

    Related items - ${related_summary(related)} + ${related_summary(related, True)}

    @@ -70,6 +70,6 @@

    Related items - + From db36db63dd994cd1ea84bf294e8b756c0bd41bab Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Wed, 2 May 2012 15:07:55 +0100 Subject: [PATCH 002/278] [2347] Added support for sorting by view count, or creation date --- ckan/config/routing.py | 2 +- ckan/controllers/related.py | 25 ++++++++++++++++++++++++- ckan/logic/action/get.py | 19 ++++++++++++++++--- ckan/model/related.py | 1 + ckan/templates/_util.html | 5 +++-- ckan/templates/related/dashboard.html | 15 ++++++++++++++- 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 1ad271e9c76..dee0f514bef 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -154,7 +154,7 @@ def make_map(): with SubMapper(map, controller='related') as m: m.connect('related_list', '/dataset/{id}/related', action='list') - m.connect('related_read', '/dataset/{id}/related/{related_id}', action='read') + m.connect('related_read', '/apps/{id}', action='read') m.connect('related_dashboard', '/apps', action='dashboard') with SubMapper(map, controller='package') as m: diff --git a/ckan/controllers/related.py b/ckan/controllers/related.py index fb6effdd5ee..3519ad26dd4 100644 --- a/ckan/controllers/related.py +++ b/ckan/controllers/related.py @@ -16,7 +16,8 @@ def dashboard(self): 'user': c.user or c.author, 'extras_as_string': True, 'for_view': True} data_dict = { - 'type_filter': base.request.params.get('type', '') + 'type_filter': base.request.params.get('type', ''), + 'sort': base.request.params.get('sort', '') } params_nopage = [(k, v) for k,v in base.request.params.items() @@ -54,6 +55,28 @@ def pager_url(q=None, page=None): return base.render( "related/dashboard.html") + def read(self, id): + context = {'model': model, 'session': model.Session, + 'user': c.user or c.author, 'extras_as_string': True, + 'for_view': True} + data_dict = {'id': id} + + try: + logic.check_access('related_show', context, data_dict) + except logic.NotAuthorized: + abort(401, _('Not authorized to see this page')) + + related = model.Session.query(model.Related).\ + filter(model.Related.id == id).first() + if not related: + abort(404, _('The requested related item was not found')) + + related.view_count += 1 + model.Session.add(related) + model.Session.commit() + + base.redirect(related.url) + def list(self, id): """ List all related items for a specific dataset """ diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index b0e315e789d..2227e4db199 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -166,9 +166,22 @@ def related_list(context, data_dict=None): related_list = [] if not dataset: related_list = model.Session.query(model.Related) - tfilter = data_dict.get('type_filter', None) - if tfilter: - related_list = related_list.filter(model.Related.type == tfilter) + + filter_on_type = data_dict.get('type_filter', None) + if filter_on_type: + related_list = related_list.filter(model.Related.type == filter_on_type) + + sort = data_dict.get('sort', None) + if sort: + sortables = { + 'view_count_asc' : model.Related.view_count.asc, + 'view_count_desc': model.Related.view_count.desc, + 'created_asc' : model.Related.created.asc, + 'created_desc': model.Related.created.desc, + } + s = sortables.get(sort, None) + if s: + related_list = related_list.order_by( s() ) else: relateds = model.Related.get_for_dataset(dataset, status='active') related_items = (r.related for r in relateds) diff --git a/ckan/model/related.py b/ckan/model/related.py index df2d5890317..c7f093ee1a2 100644 --- a/ckan/model/related.py +++ b/ckan/model/related.py @@ -16,6 +16,7 @@ meta.Column('url', meta.UnicodeText), meta.Column('created', meta.DateTime, default=datetime.datetime.now), meta.Column('owner_id', meta.UnicodeText), + meta.Column('view_count', meta.Integer, default=0) ) related_dataset_table = meta.Table('related_dataset', meta.metadata, diff --git a/ckan/templates/_util.html b/ckan/templates/_util.html index 73f976ffea1..1d75af8ae2c 100644 --- a/ckan/templates/_util.html +++ b/ckan/templates/_util.html @@ -129,15 +129,16 @@
  • - + + ${related.view_count}
    ${h.markdown_extract(related.title, extract_length=30)}
    ${h.markdown_extract(related.description, extract_length=1000)}
    No description for this item -

    View this related item

    +

    View this related item

    • diff --git a/ckan/templates/related/dashboard.html b/ckan/templates/related/dashboard.html index fc519b7baec..029b7416b32 100644 --- a/ckan/templates/related/dashboard.html +++ b/ckan/templates/related/dashboard.html @@ -25,7 +25,7 @@

    Showing items ${c.page.first_item} - ${c.p

    ${c.page.item_count} related items found

    -
    + @@ -36,6 +36,19 @@

    ${c.page.item_count} related + + + + +

    From 5a97413dd61e905624dcb1fae95d4cbe180ea5c5 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 3 May 2012 09:08:12 +0100 Subject: [PATCH 003/278] [2347] viewcount migration and related on user page --- ckan/controllers/user.py | 1 + ckan/lib/dictization/model_dictize.py | 7 +++++++ ckan/migration/versions/058_add_related_viewcount.py | 11 +++++++++++ ckan/templates/user/read.html | 8 ++++++-- 4 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 ckan/migration/versions/058_add_related_viewcount.py diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index d17df6fc410..d836a459c06 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -91,6 +91,7 @@ def read(self, id=None): except NotAuthorized: abort(401, _('Not authorized to see this page')) + context['with_related'] = True try: user_dict = get_action('user_show')(context,data_dict) except NotFound: diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 74aa8b78877..d95950c9a5f 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -355,6 +355,13 @@ def user_dictize(user, context): result_dict['number_of_edits'] = user.number_of_edits() result_dict['number_administered_packages'] = user.number_administered_packages() + model = context['model'] + session = model.Session + + if context.get('with_related'): + result_dict['related_items'] = session.query(model.Related).\ + filter(model.Related.owner_id==user.id).all() + return result_dict def task_status_dictize(task_status, context): diff --git a/ckan/migration/versions/058_add_related_viewcount.py b/ckan/migration/versions/058_add_related_viewcount.py new file mode 100644 index 00000000000..f85493ee66e --- /dev/null +++ b/ckan/migration/versions/058_add_related_viewcount.py @@ -0,0 +1,11 @@ +from sqlalchemy import * +from migrate import * + +def upgrade(migrate_engine): + migrate_engine.execute(''' + ALTER TABLE "related" + ADD COLUMN view_count int NOT NULL DEFAULT 0; + + UPDATE related SET view_count=0 WHERE view_count IS NULL; + ''' + ) diff --git a/ckan/templates/user/read.html b/ckan/templates/user/read.html index 681f0dbf701..91b14c1abee 100644 --- a/ckan/templates/user/read.html +++ b/ckan/templates/user/read.html @@ -1,7 +1,7 @@ - + ${c.user_dict['display_name']} - User ${h.linked_gravatar(c.user_dict['email_hash'],48)} @@ -14,7 +14,7 @@ - +
    @@ -58,6 +58,10 @@ ${c.user_dict['number_of_edits']} Edits

    • +
  • + ${len(c.user_dict['related_items'])} + Related items +
    • From 6002a53a6be3c9a1d9a5940a05377d7f8233d5bc Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 3 May 2012 11:39:13 +0100 Subject: [PATCH 004/278] [2347] Restrict updates/create of related featured flag to admins for now --- ckan/logic/action/create.py | 6 ++++++ ckan/logic/action/update.py | 5 +++++ ckan/logic/schema.py | 1 + ckan/templates/related/dashboard.html | 2 +- 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index aee73a32aa4..5da99b6db83 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -1,6 +1,7 @@ import logging from pylons.i18n import _ +import ckan.authz as authz import ckan.lib.plugins as lib_plugins import ckan.logic as logic import ckan.rating as ratings @@ -126,6 +127,11 @@ def related_create(context, data_dict): model.Session.rollback() raise ValidationError(errors, error_summary(errors)) + # Only sys admins can update a related item to make it 1 + if not authz.Authorizer().is_sysadmin(unicode(user)): + data['featured'] = 0 + + related = model_save.related_dict_save(data, context) if not context.get('defer_commit'): model.repo.commit_and_remove() diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index cd5aa2fe8da..3b839d0a07d 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -4,6 +4,7 @@ from pylons.i18n import _ from vdm.sqlalchemy.base import SQLAlchemySession +import ckan.authz as authz import ckan.plugins as plugins import ckan.logic as logic import ckan.logic.schema @@ -120,6 +121,10 @@ def related_update(context, data_dict): model.Session.rollback() raise ValidationError(errors, error_summary(errors)) + # Only sys admins can update a related item to make it 1 + if not authz.Authorizer().is_sysadmin(unicode(user)): + data['featured'] = 0 + related = model_save.related_dict_save(data, context) if not context.get('defer_commit'): model.repo.commit() diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 09ffe2217ae..4c174e7af27 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -245,6 +245,7 @@ def default_related_schema(): 'url': [ignore_missing, unicode], 'owner_id': [not_empty, unicode], 'created': [ignore], + 'featured': [ignore_missing, unicode], } return schema diff --git a/ckan/templates/related/dashboard.html b/ckan/templates/related/dashboard.html index aa1f0fb2fe6..39b4664be3c 100644 --- a/ckan/templates/related/dashboard.html +++ b/ckan/templates/related/dashboard.html @@ -14,7 +14,7 @@ no-sidebar - Apps + Application Applications From 7cc252d7d5d3c3b7a892f756e8443cc599e7de3c Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 3 May 2012 14:10:26 +0100 Subject: [PATCH 005/278] [2347] Added activity for creating of related items --- ckan/logic/action/create.py | 23 ++++++++++++++++++- ckan/logic/action/get.py | 6 +++++ ckan/logic/validators.py | 15 ++++++++++++ .../activity_streams/new_related_item.html | 13 +++++++++++ ckan/templates/related/add-related.html | 2 +- ckan/templates/related/dashboard.html | 2 +- 6 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 ckan/templates/activity_streams/new_related_item.html diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 5da99b6db83..84f99211bd2 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -116,6 +116,7 @@ def resource_create(context, data_dict): def related_create(context, data_dict): model = context['model'] + session = context['session'] user = context['user'] userobj = model.User.get(user) @@ -141,10 +142,30 @@ def related_create(context, data_dict): dataset.related.append( related ) model.repo.commit_and_remove() + session.flush() + + related_dict = model_dictize.related_dictize(related, context) + activity_dict = { + 'user_id': userobj.id, + 'object_id': related.id, + 'activity_type': 'new related item', + } + activity_dict['data'] = { + 'related': related_dict + } + activity_create_context = { + 'model': model, + 'user': user, + 'defer_commit':True, + 'session': session + } + activity_create(activity_create_context, activity_dict, ignore_auth=True) + session.commit() + context["related"] = related context["id"] = related.id log.debug('Created object %s' % str(related.title)) - return model_dictize.related_dictize(related, context) + return related_dict def package_relationship_create(context, data_dict): diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 4d26f89ccab..0cc18d77891 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1245,6 +1245,11 @@ def render_new_package_activity(context, activity): return render('activity_streams/new_package.html', extra_vars = {'activity': activity}) +def render_new_related_activity(context, activity): + return render('activity_streams/new_related_item.html', + extra_vars = {'activity': activity, + 'type': activity['data']['related']['type']}) + def render_deleted_package_activity(context, activity): return render('activity_streams/deleted_package.html', extra_vars = {'activity': activity}) @@ -1347,6 +1352,7 @@ def render_deleted_group_activity(context, activity): 'new group' : render_new_group_activity, 'changed group' : render_changed_group_activity, 'deleted group' : render_deleted_group_activity, + 'new related item': render_new_related_activity } def _activity_list_to_html(context, activity_stream): diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 58c69ee3aef..3ca2e100529 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -114,6 +114,20 @@ def group_id_exists(group_id, context): raise Invalid('%s: %s' % (_('Not found'), _('Group'))) return group_id + +def related_id_exists(related_id, context): + """Raises Invalid if the given related_id does not exist in the model + given in the context, otherwise returns the given related_id. + + """ + model = context['model'] + session = context['session'] + + result = session.query(model.Related).get(related_id) + if not result: + raise Invalid('%s: %s' % (_('Not found'), _('Related'))) + return related_id + def group_id_or_name_exists(reference, context): """ Raises Invalid if a group identified by the name or id cannot be found. @@ -146,6 +160,7 @@ def activity_type_exists(activity_type): 'new group' : group_id_exists, 'changed group' : group_id_exists, 'deleted group' : group_id_exists, + 'new related item': related_id_exists } def object_id_validator(key, activity_dict, errors, context): diff --git a/ckan/templates/activity_streams/new_related_item.html b/ckan/templates/activity_streams/new_related_item.html new file mode 100644 index 00000000000..be0690782f7 --- /dev/null +++ b/ckan/templates/activity_streams/new_related_item.html @@ -0,0 +1,13 @@ + +${ h.activity_div( + template=_("{actor} created the %s {object}"), + activity=activity, + actor=h.linked_user(activity.user_id), + object=h.link_to(activity.data.related['title'], h.url_for(controller='related', action='read', id=activity.data.related['id'])) + ) % type} + diff --git a/ckan/templates/related/add-related.html b/ckan/templates/related/add-related.html index e5b2a64a1a3..fdda87d0ee3 100644 --- a/ckan/templates/related/add-related.html +++ b/ckan/templates/related/add-related.html @@ -21,7 +21,7 @@

    Add related item

    diff --git a/ckan/templates/related/dashboard.html b/ckan/templates/related/dashboard.html index 39b4664be3c..7149cd6b58d 100644 --- a/ckan/templates/related/dashboard.html +++ b/ckan/templates/related/dashboard.html @@ -31,7 +31,7 @@

    ${c.page.item_count} related From 3eac5416da57d42a4b5ee9ae48b30efffbfc9f6b Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 3 May 2012 15:30:07 +0100 Subject: [PATCH 006/278] [2347] Added activity for deleting related items --- ckan/logic/action/delete.py | 25 +++++++++++++++++++ ckan/logic/action/get.py | 7 +++++- ckan/logic/validators.py | 3 ++- .../deleted_related_item.html | 14 +++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 ckan/templates/activity_streams/deleted_related_item.html diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index c806a5113cd..1084f4707ec 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -3,11 +3,13 @@ import ckan.logic import ckan.logic.action import ckan.plugins as plugins +import ckan.lib.dictization.model_dictize as model_dictize # define some shortcuts ValidationError = ckan.logic.ValidationError NotFound = ckan.logic.NotFound check_access = ckan.logic.check_access +get_action = ckan.logic.get_action def package_delete(context, data_dict): @@ -65,8 +67,12 @@ def package_relationship_delete(context, data_dict): model.repo.commit() def related_delete(context, data_dict): + import ckan.logic.action as action + model = context['model'] + session = context['session'] user = context['user'] + userobj = model.User.get(user) id = data_dict['id'] entity = model.Related.get(id) @@ -76,6 +82,25 @@ def related_delete(context, data_dict): check_access('related_delete',context, data_dict) + related_dict = model_dictize.related_dictize(entity, context) + activity_dict = { + 'user_id': userobj.id, + 'object_id': entity.id, + 'activity_type': 'deleted related item', + } + activity_dict['data'] = { + 'related': related_dict + } + activity_create_context = { + 'model': model, + 'user': user, + 'defer_commit':True, + 'session': session + } + + get_action('activity_create')(activity_create_context, activity_dict, ignore_auth=True) + session.commit() + entity.delete() model.repo.commit() diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 0cc18d77891..a45b90c63f3 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1262,6 +1262,10 @@ def render_changed_resource_activity(context, activity, detail): return render('activity_streams/changed_resource.html', extra_vars = {'activity': activity, 'detail': detail}) +def render_deleted_related_activity(context, activity): + return render('activity_streams/deleted_related_item.html', + extra_vars = {'activity': activity}) + def render_deleted_resource_activity(context, activity, detail): return render('activity_streams/deleted_resource.html', extra_vars = {'activity': activity, 'detail': detail}) @@ -1352,7 +1356,8 @@ def render_deleted_group_activity(context, activity): 'new group' : render_new_group_activity, 'changed group' : render_changed_group_activity, 'deleted group' : render_deleted_group_activity, - 'new related item': render_new_related_activity + 'new related item': render_new_related_activity, + 'deleted related item': render_deleted_related_activity } def _activity_list_to_html(context, activity_stream): diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 3ca2e100529..3f58a78df37 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -160,7 +160,8 @@ def activity_type_exists(activity_type): 'new group' : group_id_exists, 'changed group' : group_id_exists, 'deleted group' : group_id_exists, - 'new related item': related_id_exists + 'new related item': related_id_exists, + 'deleted related item': related_id_exists } def object_id_validator(key, activity_dict, errors, context): diff --git a/ckan/templates/activity_streams/deleted_related_item.html b/ckan/templates/activity_streams/deleted_related_item.html new file mode 100644 index 00000000000..bf615e1ad2c --- /dev/null +++ b/ckan/templates/activity_streams/deleted_related_item.html @@ -0,0 +1,14 @@ + +${h.activity_div( + template=_("{actor} deleted the related item {object}"), + activity=activity, + actor=h.linked_user(activity.user_id), + object=activity.data.related.title, + )} + From da04077f943dfd83354b74097c5aa09d9ef7fcbc Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 4 May 2012 09:56:17 +0100 Subject: [PATCH 007/278] [2347] Added config option to hide related and documented it --- ckan/templates/package/layout.html | 3 +-- doc/configuration.rst | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ckan/templates/package/layout.html b/ckan/templates/package/layout.html index 3646d0f5a5d..ae63355a17f 100644 --- a/ckan/templates/package/layout.html +++ b/ckan/templates/package/layout.html @@ -34,8 +34,7 @@ -
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
    • - +
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
  • |
    • diff --git a/doc/configuration.rst b/doc/configuration.rst index 7bcc74701e3..728cbecda24 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -122,6 +122,17 @@ This sets a space-separated list of extra field key values which will not be sho .. index:: single: rdf_packages +package_show_related +^^^^^^^^^^^^^^^^^^^^ + +package_show_related:: + + package_show_related = 0 + +Default value: 1 + +When set to 0 this setting will hide the related item tab on the package read page. If the value is not set, or is set to 1, then the related item tab will shown. + rdf_packages ^^^^^^^^^^^^ From eeb1cc954e0cc7604edbdcc6fb3a95d3b91c5f73 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 4 May 2012 10:51:14 +0100 Subject: [PATCH 008/278] [2347] Add a url validator to the related schema --- ckan/logic/schema.py | 7 ++++--- ckan/logic/validators.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 4c174e7af27..8bcbdfd998b 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -37,7 +37,8 @@ user_id_exists, object_id_validator, activity_type_exists, - tag_not_in_vocabulary) + tag_not_in_vocabulary, + url_validator) from formencode.validators import OneOf import ckan.model @@ -241,8 +242,8 @@ def default_related_schema(): 'title': [not_empty, unicode], 'description': [ignore_missing, unicode], 'type': [not_empty, unicode], - 'image_url': [ignore_missing, unicode], - 'url': [ignore_missing, unicode], + 'image_url': [ignore_missing, unicode, url_validator], + 'url': [ignore_missing, unicode, url_validator], 'owner_id': [not_empty, unicode], 'created': [ignore], 'featured': [ignore_missing, unicode], diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index 3f58a78df37..9451d4d06db 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -492,3 +492,23 @@ def tag_not_in_vocabulary(key, tag_dict, errors, context): (tag_name, vocabulary_id)) else: return + +def url_validator(key, data, errors, context): + """ Checks that the provided value (if it is present) is a valid URL """ + import urlparse + import string + + model = context['model'] + session = context['session'] + + url = data.get(key, None) + if not url: + return + + pieces = urlparse.urlparse(url) + if all([pieces.scheme, pieces.netloc]) and \ + set(pieces.netloc) <= set(string.letters + string.digits + '-.') and \ + pieces.scheme in ['http', 'https']: + return + + errors[key].append(_('Please provide a valid URL')) From d2ea8735a3d01a8135c7635b83f204c92712b878 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 4 May 2012 11:05:53 +0100 Subject: [PATCH 009/278] [2347] Following advice from tobes to use h.bool to allow use or more than 1 or 0 in ini file --- ckan/templates/package/layout.html | 2 +- doc/configuration.rst | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/templates/package/layout.html b/ckan/templates/package/layout.html index ae63355a17f..5ff630a88ef 100644 --- a/ckan/templates/package/layout.html +++ b/ckan/templates/package/layout.html @@ -34,7 +34,7 @@ -
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
    • +
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
  • |
    • diff --git a/doc/configuration.rst b/doc/configuration.rst index 728cbecda24..523323c0ce2 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -127,11 +127,11 @@ package_show_related package_show_related:: - package_show_related = 0 + package_show_related = false -Default value: 1 +Default value: true -When set to 0 this setting will hide the related item tab on the package read page. If the value is not set, or is set to 1, then the related item tab will shown. +When set to false, or no, this setting will hide the related item tab on the package read page. If the value is not set, or is set to true or yes, then the related item tab will shown. rdf_packages ^^^^^^^^^^^^ From 45336e342fc1a0dd6a469d67c0438296a2dc6513 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 4 May 2012 11:14:37 +0100 Subject: [PATCH 010/278] [2347] Fixing up naming --- ckan/templates/package/layout.html | 2 +- doc/configuration.rst | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/templates/package/layout.html b/ckan/templates/package/layout.html index 5ff630a88ef..f196b37e9e7 100644 --- a/ckan/templates/package/layout.html +++ b/ckan/templates/package/layout.html @@ -34,7 +34,7 @@ -
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
    • +
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
  • |
    • diff --git a/doc/configuration.rst b/doc/configuration.rst index 523323c0ce2..ac2f45e20e1 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -122,12 +122,12 @@ This sets a space-separated list of extra field key values which will not be sho .. index:: single: rdf_packages -package_show_related -^^^^^^^^^^^^^^^^^^^^ +ckan.dataset.show_related +^^^^^^^^^^^^^^^^^^^^^^^^^ -package_show_related:: +ckan.dataset.show_related:: - package_show_related = false + ckan.dataset.show_related = false Default value: true From 8c73fbcda5de09db39efb4469c4e22b0812db87d Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Wed, 9 May 2012 10:46:34 +0100 Subject: [PATCH 011/278] [2347] Use auth check in template for add button and make sure related_create for publisher auth is accurate --- ckan/logic/action/create.py | 3 ++- ckan/logic/auth/publisher/create.py | 16 +++++++++++++--- ckan/templates/related/related_list.html | 4 ++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 0bc3110016a..2b44b5e2884 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -121,6 +121,8 @@ def related_create(context, data_dict): user = context['user'] userobj = model.User.get(user) + check_access('related_create', context, data_dict) + data_dict["owner_id"] = userobj.id data, errors = validate(data_dict, ckan.logic.schema.default_related_schema(), @@ -133,7 +135,6 @@ def related_create(context, data_dict): if not authz.Authorizer().is_sysadmin(unicode(user)): data['featured'] = 0 - related = model_save.related_dict_save(data, context) if not context.get('defer_commit'): model.repo.commit_and_remove() diff --git a/ckan/logic/auth/publisher/create.py b/ckan/logic/auth/publisher/create.py index d1bece7ebaf..a5112a599fe 100644 --- a/ckan/logic/auth/publisher/create.py +++ b/ckan/logic/auth/publisher/create.py @@ -25,10 +25,20 @@ def related_create(context, data_dict=None): user = context['user'] userobj = model.User.get( user ) - if userobj: - return {'success': True} + if not userobj: + return {'success': False, 'msg': _('You must be logged in to add a related item')} + + if 'dataset_id' in data_dict: + # If this is to be associated with a dataset then we need to make sure that + # the user doing so is a member of that group + dataset = model.Package.get(data_dict['dataset_id']) + if dataset and not _groups_intersect( userobj.get_groups(), + dataset.get_groups() ): + return {'success': False, + 'msg': _('You do not have permission to create an item')} + + return {'success': True } - return {'success': False, 'msg': _('You must be logged in to add a related item')} def resource_create(context, data_dict): diff --git a/ckan/templates/related/related_list.html b/ckan/templates/related/related_list.html index 4d18838fc28..f97872fb440 100644 --- a/ckan/templates/related/related_list.html +++ b/ckan/templates/related/related_list.html @@ -23,10 +23,10 @@
    ${add_related(c.pkg)} -

    Related items Add related item

    +

    Related items Add related item

    - There are no related items here yet, why not add one? + There are no related items here yet, why not add one?
    diff --git a/ckan/templates/package/layout.html b/ckan/templates/package/layout.html index 24b340bcc6f..a5425a4031c 100644 --- a/ckan/templates/package/layout.html +++ b/ckan/templates/package/layout.html @@ -34,7 +34,7 @@ -
  • ${h.subnav_link(h.icon('package') + _('Related') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
    • +
  • ${h.subnav_link(h.icon('package') + _('Apps & Ideas') + ' (%s)' % c.related_count, controller='related', action='list', id=c.pkg.name)}
  • ${h.subnav_link(h.icon('page_stack') + _('History'), controller='package', action='history', id=c.pkg.name)}
  • ${h.subnav_link( diff --git a/ckan/templates/related/add-related.html b/ckan/templates/related/add-related.html index f7396720d8b..1e5f23e49d5 100644 --- a/ckan/templates/related/add-related.html +++ b/ckan/templates/related/add-related.html @@ -9,7 +9,7 @@ py:def="add_related(dataset)">
    × -

    Add related item

    +

    Add item

    • - ${facet_div('tags', 'Tags')} - ${facet_div('res_format', 'Resource Formats')} + ${facet_div('tags', _('Tags'))} + ${facet_div('res_format', _('Resource Formats'))} From 4594319109fd4e2cd8f152216ae9214e22a8c2ca Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 2 Jul 2012 15:38:18 +0200 Subject: [PATCH 073/278] Add strings from core extensions to pot file --- setup.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a8f8297e035..74187be3b0c 100644 --- a/setup.py +++ b/setup.py @@ -45,8 +45,13 @@ }), ('public/**', 'ignore', None), ], - 'ckanext/stats/templates': [ + 'ckanext': [ + ('**.py', 'python', None), ('**.html', 'genshi', None), + ('multilingual/solr/*.txt', 'ignore', None), + ('**.txt', 'genshi', { + 'template_class': 'genshi.template:TextTemplate' + }), ]}, entry_points=""" [nose.plugins.0.10] From fb782e2043d58c7015eb17ee6fe236316dc79ea9 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 2 Jul 2012 15:38:52 +0200 Subject: [PATCH 074/278] Fix a string that had a lot of whitespace in it --- ckanext/organizations/controllers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/organizations/controllers.py b/ckanext/organizations/controllers.py index fbe6e25e464..371b2141d98 100644 --- a/ckanext/organizations/controllers.py +++ b/ckanext/organizations/controllers.py @@ -95,8 +95,8 @@ def apply(self, id=None, data=None, errors=None, error_summary=None): def _add_users( self, group, parameters ): if not group: - h.flash_error(_("There was a problem with your submission, \ - please correct it and try again")) + h.flash_error(_("There was a problem with your submission, " + "please correct it and try again")) errors = {"reason": ["No reason was supplied"]} return self.apply(group.id, errors=errors, error_summary=action.error_summary(errors)) From 58d054970a1ff108678ae8ee3124c7f27eca224a Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Mon, 25 Jun 2012 17:42:38 +0100 Subject: [PATCH 075/278] Dictize results list of resource_search action. Looks like this action wasn't even working! --- ckan/logic/action/get.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index f15525bd768..cb215d0546b 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1240,7 +1240,8 @@ def resource_search(context, data_dict): else: results.append(result) - return {'count': count, 'results': results} + return {'count': count, + 'results': model_dictize.resource_list_dictize(results, context)} def _tag_search(context, data_dict): model = context['model'] From f386122015e1b8408b1f94046d8422eb2d17ac7a Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Mon, 25 Jun 2012 17:44:20 +0100 Subject: [PATCH 076/278] Docstring for resource_search action. --- ckan/logic/action/get.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index cb215d0546b..1737b76bb17 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1178,18 +1178,24 @@ def package_search(context, data_dict): def resource_search(context, data_dict): ''' + Searches for resources satisfying a given search criteria. + + It returns a dictionary with 2 fields: ``count`` and ``results``. The + ``count`` field contains the total number of Resources found without the + limit or query parameters having an effect. The ``results`` field is a + list of dictized Resource objects. :param fields: :type fields: - :param order_by: - :type order_by: - :param offset: - :type offset: - :param limit: - :type limit: - - :returns: - :rtype: + :param order_by: A field on the Resource model that orders the results. + :type order_by: string + :param offset: Apply an offset to the query. + :type offset: int + :param limit: Apply a limit to the query. + :type limit: int + + :returns: A dictionary with a ``count`` field, and a ``results`` field. + :rtype: dict ''' model = context['model'] From ebd9bc613ea4cea5617998cf2c669ff3f267f4dd Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 15:43:55 +0100 Subject: [PATCH 077/278] Add 'q' parameter to resource_search action. This deprecates the existing 'fields' parameter, which is not compatible with GET-ing the resource_search action. --- ckan/logic/action/get.py | 56 +++++++++++++++++++++++++++-- ckan/tests/logic/test_action.py | 63 +++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 1737b76bb17..37d0ac0a37e 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1185,8 +1185,52 @@ def resource_search(context, data_dict): limit or query parameters having an effect. The ``results`` field is a list of dictized Resource objects. - :param fields: - :type fields: + The 'q' parameter is a required field. It is a string of the form + ``{field}:{term}`` or a list of strings, each of the same form. Within + each string, ``{field}`` is a field or extra field on the Resource domain + object. + + If ``{field}`` is ``"hash"``, then an attempt is made to match the + `{term}` as a *prefix* of the ``Resource.hash`` field. + + If ``{field}`` is an extra field, then an attempt is made to match against + the extra fields stored against the Resource. + + Note: The search is limited to search against extra fields declared in + the config setting ``ckan.extra_resource_fields``. + + Note: Due to a Resource's extra fields being stored as a json blob, the + match is made against the json string representation. As such, false + positives may occur: + + If the search criteria is: :: + + query = "field1:term1" + + Then a json blob with the string representation of: :: + + {"field1": "foo", "field2": "term1"} + + will match the search criteria! This is a known short-coming of this + approach. + + All matches are made ignoring case; and apart from the ``"hash"`` field, + a term matches if it is a substring of the field's value. + + Finally, when specifying more than one search criteria, the criteria are + AND-ed together. + + The ``order`` parameter is used to control the ordering of the results. + Currently only ordering one field is available, and in ascending order + only. + + The ``fields`` parameter is deprecated as it is not compatible with calling + this action with a GET request to the action API. + + :param query: The search criteria. See above for description. + :type query: string or list of strings of the form "{field}:{term1}" + :param fields: Deprecated + :type fields: dict of fields to search terms. :param order_by: A field on the Resource model that orders the results. :type order_by: string :param offset: Apply an offset to the query. @@ -1201,15 +1245,21 @@ def resource_search(context, data_dict): model = context['model'] session = context['session'] + query = _get_or_bust(data_dict, 'query') fields = _get_or_bust(data_dict, 'fields') order_by = data_dict.get('order_by') offset = data_dict.get('offset') limit = data_dict.get('limit') + if isinstance(query, basestring): + query = [query] + + # TODO: escape ':' with '\:' + fields = dict(pair.split(":", 1) for pair in query) + # TODO: should we check for user authentication first? q = model.Session.query(model.Resource) resource_fields = model.Resource.get_columns() - for field, terms in fields.items(): if isinstance(terms, basestring): terms = terms.split() diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index d104c5d3e94..c8dc581c341 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1023,6 +1023,69 @@ def test_41_missing_action(self): except KeyError: assert True + def test_42_resource_search_with_single_field_query(self): + request_body = { + 'q': ["description:index"], + 'fields': [] + } + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + result = json.loads(response.body)['result']['results'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert result is not [] + for resource in result: + assert "index" in resource['description'].lower() + + def test_42_resource_search_across_multiple_fields(self): + request_body = { + 'q': ["description:index", "format:json"], + 'fields': [] + } + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + result = json.loads(response.body)['result']['results'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert result is not [] + for resource in result: + assert "index" in resource['description'].lower() + assert "json" in resource['format'].lower() + + def test_42_resource_search_test_percentage_is_escaped(self): + pass + + def test_42_resource_search_escaped_colons(self): + pass + + def test_42_resource_search_fields_parameter_still_accepted(self): + '''The fields parameter is deprecated, but check it still works. + + Remove this test when removing the fields parameter. (#????) + ''' + request_body = { + 'fields': [("description", "index")], + 'q': [], + } + + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + result = json.loads(response.body)['result']['results'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert result is not [] + for resource in result: + assert "index" in resource['description'].lower() + class TestActionTermTranslation(WsgiAppCase): @classmethod From cc070212a450b7c716cf79e81b29ba54301709be Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 16:52:49 +0100 Subject: [PATCH 078/278] Ensure legacy parameter, fields, still works. Keep the old fields `parameter's` behaviour, but encourage use of the new `query` parameter. --- ckan/logic/action/get.py | 38 +++++++++++++++++++++++++-------- ckan/tests/logic/test_action.py | 9 +++----- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 37d0ac0a37e..c0657748ce9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1245,24 +1245,44 @@ def resource_search(context, data_dict): model = context['model'] session = context['session'] - query = _get_or_bust(data_dict, 'query') - fields = _get_or_bust(data_dict, 'fields') + # Allow either query or fields parameter to be given, but not both. + # Once ``fields`` parameter is dropped, this can be made simpler. + # The result of all this gumpf is to populate the local `fields` variable + # with mappings from field names to list of search terms, or a single + # search-term string. + query = data_dict.get('query') + fields = data_dict.get('fields') + if query is None and fields is None: + raise ValidationError({'query': _('Missing value')}) + elif query is not None and fields is not None: + raise ValidationError( + {'fields': _('Do not specify if using "query" parameter')}) + elif query is not None: + if isinstance(query, basestring): + query = [query] + + # TODO: escape ':' with '\:' + fields = dict(pair.split(":", 1) for pair in query) + else: + # legacy fields paramter would split string terms + # maintain that behaviour + split_terms = {} + for field, terms in fields.items(): + if isinstance(terms, basestring): + terms = terms.split() + split_terms[field] = terms + fields = split_terms + order_by = data_dict.get('order_by') offset = data_dict.get('offset') limit = data_dict.get('limit') - if isinstance(query, basestring): - query = [query] - - # TODO: escape ':' with '\:' - fields = dict(pair.split(":", 1) for pair in query) - # TODO: should we check for user authentication first? q = model.Session.query(model.Resource) resource_fields = model.Resource.get_columns() for field, terms in fields.items(): if isinstance(terms, basestring): - terms = terms.split() + terms = [terms] if field not in resource_fields: raise search.SearchError('Field "%s" not recognised in Resource search.' % field) for term in terms: diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index c8dc581c341..acb4104d964 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1025,8 +1025,7 @@ def test_41_missing_action(self): def test_42_resource_search_with_single_field_query(self): request_body = { - 'q': ["description:index"], - 'fields': [] + 'query': ["description:index"], } postparams = json.dumps(request_body) response = self.app.post('/api/action/resource_search', @@ -1042,8 +1041,7 @@ def test_42_resource_search_with_single_field_query(self): def test_42_resource_search_across_multiple_fields(self): request_body = { - 'q': ["description:index", "format:json"], - 'fields': [] + 'query': ["description:index", "format:json"], } postparams = json.dumps(request_body) response = self.app.post('/api/action/resource_search', @@ -1070,8 +1068,7 @@ def test_42_resource_search_fields_parameter_still_accepted(self): Remove this test when removing the fields parameter. (#????) ''' request_body = { - 'fields': [("description", "index")], - 'q': [], + 'fields': {"description": "index"}, } postparams = json.dumps(request_body) From f8450f4f2b11308e340c83bd78d114e513717615 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 17:42:28 +0100 Subject: [PATCH 079/278] Special chars are escaped in search terms. --- ckan/logic/action/get.py | 1 + ckan/tests/logic/test_action.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index c0657748ce9..5c03b07b63e 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1286,6 +1286,7 @@ def resource_search(context, data_dict): if field not in resource_fields: raise search.SearchError('Field "%s" not recognised in Resource search.' % field) for term in terms: + term = misc.escape_sql_like_special_characters(term) model_attr = getattr(model.Resource, field) if field == 'hash': q = q.filter(model_attr.ilike(unicode(term) + '%')) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index acb4104d964..330238aa661 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1031,11 +1031,12 @@ def test_42_resource_search_with_single_field_query(self): response = self.app.post('/api/action/resource_search', params=postparams) result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] ## Due to the side-effect of previously run tests, there may be extra ## resources in the results. So just check that each found Resource ## matches the search criteria - assert result is not [] + assert count > 0 for resource in result: assert "index" in resource['description'].lower() @@ -1047,20 +1048,29 @@ def test_42_resource_search_across_multiple_fields(self): response = self.app.post('/api/action/resource_search', params=postparams) result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] ## Due to the side-effect of previously run tests, there may be extra ## resources in the results. So just check that each found Resource ## matches the search criteria - assert result is not [] + assert count > 0 for resource in result: assert "index" in resource['description'].lower() assert "json" in resource['format'].lower() def test_42_resource_search_test_percentage_is_escaped(self): - pass + request_body = { + 'query': ["description:index%"], + } + postparams = json.dumps(request_body) + response = self.app.post('/api/action/resource_search', + params=postparams) + count = json.loads(response.body)['result']['count'] - def test_42_resource_search_escaped_colons(self): - pass + # There shouldn't be any results. If the '%' character wasn't + # escaped correctly, then the search would match because of the + # unescaped wildcard. + assert count is 0 def test_42_resource_search_fields_parameter_still_accepted(self): '''The fields parameter is deprecated, but check it still works. @@ -1075,11 +1085,12 @@ def test_42_resource_search_fields_parameter_still_accepted(self): response = self.app.post('/api/action/resource_search', params=postparams) result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] ## Due to the side-effect of previously run tests, there may be extra ## resources in the results. So just check that each found Resource ## matches the search criteria - assert result is not [] + assert count > 0 for resource in result: assert "index" in resource['description'].lower() From aeef82cbf16656308730b3e3b9121a4828b526c2 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 17:53:41 +0100 Subject: [PATCH 080/278] Useful validation errors for resource_search --- ckan/logic/action/get.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 5c03b07b63e..69ad1cd4949 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1260,9 +1260,11 @@ def resource_search(context, data_dict): elif query is not None: if isinstance(query, basestring): query = [query] - - # TODO: escape ':' with '\:' - fields = dict(pair.split(":", 1) for pair in query) + try: + fields = dict(pair.split(":", 1) for pair in query) + except ValueError: + raise ValidationError( + {'query': _('Must be {field}:{value} pair(s)')}) else: # legacy fields paramter would split string terms # maintain that behaviour @@ -1284,7 +1286,10 @@ def resource_search(context, data_dict): if isinstance(terms, basestring): terms = [terms] if field not in resource_fields: - raise search.SearchError('Field "%s" not recognised in Resource search.' % field) + raise ValidationError( + {'query': + _('Field "{field}" not recognised in resource_search.')\ + .format(field=field)}) for term in terms: term = misc.escape_sql_like_special_characters(term) model_attr = getattr(model.Resource, field) From 966ab39183351701656786d63a193bf79898c748 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 17:54:23 +0100 Subject: [PATCH 081/278] Remove unused session object. --- ckan/logic/action/get.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 69ad1cd4949..fc7ca2a1090 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1243,7 +1243,6 @@ def resource_search(context, data_dict): ''' model = context['model'] - session = context['session'] # Allow either query or fields parameter to be given, but not both. # Once ``fields`` parameter is dropped, this can be made simpler. From 79505e541263eadce6f7245e6548240499413ca1 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:00:48 +0100 Subject: [PATCH 082/278] General tidy of resource_search action. --- ckan/logic/action/get.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index fc7ca2a1090..82963437153 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1244,18 +1244,21 @@ def resource_search(context, data_dict): ''' model = context['model'] - # Allow either query or fields parameter to be given, but not both. - # Once ``fields`` parameter is dropped, this can be made simpler. + # Allow either the `query` or `fields` parameter to be given, but not both. + # Once `fields` parameter is dropped, this can be made simpler. # The result of all this gumpf is to populate the local `fields` variable # with mappings from field names to list of search terms, or a single # search-term string. query = data_dict.get('query') fields = data_dict.get('fields') + if query is None and fields is None: raise ValidationError({'query': _('Missing value')}) + elif query is not None and fields is not None: raise ValidationError( {'fields': _('Do not specify if using "query" parameter')}) + elif query is not None: if isinstance(query, basestring): query = [query] @@ -1264,9 +1267,10 @@ def resource_search(context, data_dict): except ValueError: raise ValidationError( {'query': _('Must be {field}:{value} pair(s)')}) + else: - # legacy fields paramter would split string terms - # maintain that behaviour + # The legacy fields paramter splits string terms. + # So maintain that behaviour split_terms = {} for field, terms in fields.items(): if isinstance(terms, basestring): @@ -1282,18 +1286,29 @@ def resource_search(context, data_dict): q = model.Session.query(model.Resource) resource_fields = model.Resource.get_columns() for field, terms in fields.items(): + if isinstance(terms, basestring): terms = [terms] + if field not in resource_fields: raise ValidationError( {'query': _('Field "{field}" not recognised in resource_search.')\ .format(field=field)}) + for term in terms: + + # prevent pattern injection term = misc.escape_sql_like_special_characters(term) + model_attr = getattr(model.Resource, field) + + # Treat the has field separately, see docstring. if field == 'hash': q = q.filter(model_attr.ilike(unicode(term) + '%')) + + # Resource extras are stored in a json blob. So searching for + # matching fields is a bit trickier. See the docstring. elif field in model.Resource.get_extra_columns(): model_attr = getattr(model.Resource, 'extras') @@ -1302,6 +1317,8 @@ def resource_search(context, data_dict): model_attr.ilike(u'''%%"%s": "%%%s%%"}''' % (field, term)) ) q = q.filter(like) + + # Just a regular field else: q = q.filter(model_attr.ilike('%' + unicode(term) + '%')) From d4ea63a2b9c2f5dd339879269fb366f847b3ab76 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:05:41 +0100 Subject: [PATCH 083/278] Add reference to ticket number --- ckan/tests/logic/test_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 330238aa661..c9b672126b6 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1075,7 +1075,7 @@ def test_42_resource_search_test_percentage_is_escaped(self): def test_42_resource_search_fields_parameter_still_accepted(self): '''The fields parameter is deprecated, but check it still works. - Remove this test when removing the fields parameter. (#????) + Remove this test when removing the fields parameter. (#2603) ''' request_body = { 'fields': {"description": "index"}, From dcca75713e44763645c2190cd531b30f8c99ff31 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:18:15 +0100 Subject: [PATCH 084/278] Add log message warning of deprecated parameter. Obviously this won't indicate to the action api users, only when in library access. --- ckan/logic/action/get.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 82963437153..b8825331da9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1269,6 +1269,9 @@ def resource_search(context, data_dict): {'query': _('Must be {field}:{value} pair(s)')}) else: + log.warning('Use of the "fields" parameter in resource_search is ' + 'deprecated. Use the "query" parameter instead') + # The legacy fields paramter splits string terms. # So maintain that behaviour split_terms = {} From 575a96470a9278dc660144743999b9c2849e114d Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 18:50:28 +0100 Subject: [PATCH 085/278] Context determines resource_search's result type. This is provided as it allows the search api to continue to use this action, and still provide the required result-type for the action api. --- ckan/lib/search/query.py | 6 +++++- ckan/logic/action/get.py | 25 ++++++++++++++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index a3b7adf35fe..4d11961d01d 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -186,7 +186,11 @@ def run(self, fields={}, options=None, **kwargs): else: options.update(kwargs) - context = {'model':model, 'session': model.Session} + context = { + 'model':model, + 'session': model.Session, + 'search_query': True, + } data_dict = { 'fields': fields, 'offset': options.get('offset'), diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index b8825331da9..d84cc1cd759 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1227,6 +1227,11 @@ def resource_search(context, data_dict): The ``fields`` parameter is deprecated as it is not compatible with calling this action with a GET request to the action API. + The context may contain a flag, `search_query`, which if True will make + this action behave as if being used by the internal search api. ie - the + results will not be dictized, and SearchErrors are thrown for bad search + queries (rather than ValidationErrors). + :param query: The search criteria. See above for description. :type query: string or list of strings of the form "{field}:{term1}" :param fields: Deprecated @@ -1294,10 +1299,16 @@ def resource_search(context, data_dict): terms = [terms] if field not in resource_fields: - raise ValidationError( - {'query': - _('Field "{field}" not recognised in resource_search.')\ - .format(field=field)}) + msg = _('Field "{field}" not recognised in resource_search.')\ + .format(field=field) + + # Running in the context of the internal search api. + if context.get('search_query', False): + raise search.SearchError(msg) + + # Otherwise, assume we're in the context of an external api + # and need to provide meaningful external error messages. + raise ValidationError({'query': msg}) for term in terms: @@ -1341,8 +1352,12 @@ def resource_search(context, data_dict): else: results.append(result) + # If run in the context of a search query, then don't dictize the results. + if not context.get('search_query', False): + results = model_dictize.resource_list_dictize(results, context) + return {'count': count, - 'results': model_dictize.resource_list_dictize(results, context)} + 'results': results} def _tag_search(context, data_dict): model = context['model'] From aad834c25ce6f28cd719a59d3d89f5cc79ab533c Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 26 Jun 2012 19:13:22 +0100 Subject: [PATCH 086/278] Test resource_search works with a GET request --- ckan/tests/logic/test_action.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index c9b672126b6..3b3566fecf7 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1094,6 +1094,21 @@ def test_42_resource_search_fields_parameter_still_accepted(self): for resource in result: assert "index" in resource['description'].lower() + def test_42_resource_search_accessible_via_get_request(self): + response = self.app.get('/api/action/resource_search' + '?query=description:index&query=format:json') + + result = json.loads(response.body)['result']['results'] + count = json.loads(response.body)['result']['count'] + + ## Due to the side-effect of previously run tests, there may be extra + ## resources in the results. So just check that each found Resource + ## matches the search criteria + assert count > 0 + for resource in result: + assert "index" in resource['description'].lower() + assert "json" in resource['format'].lower() + class TestActionTermTranslation(WsgiAppCase): @classmethod From f4b6a025618d4ae874f4ffb452365cc6301b4aa6 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 27 Jun 2012 10:54:35 +0100 Subject: [PATCH 087/278] Use the new 'query' parameter in resource_search instead of the now deprecated 'fields' paramter. --- ckan/lib/search/query.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 4d11961d01d..1380e531bb1 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -191,8 +191,18 @@ def run(self, fields={}, options=None, **kwargs): 'session': model.Session, 'search_query': True, } + + # Transform fields into structure required by the resource_search + # action. + query = [] + for field, terms in fields.items(): + if isinstance(terms, basestring): + terms = terms.split() + for term in terms: + query.append(':'.join([field, term])) + data_dict = { - 'fields': fields, + 'query': query, 'offset': options.get('offset'), 'limit': options.get('limit'), 'order_by': options.get('order_by') From 80781b39423084f33cacd981e8e88e9a69c4d1ce Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Wed, 27 Jun 2012 10:56:36 +0100 Subject: [PATCH 088/278] Clarify translatable string. ``{field}`` is usually used to represent a substitution. --- ckan/logic/action/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index d84cc1cd759..829a7906828 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1271,7 +1271,7 @@ def resource_search(context, data_dict): fields = dict(pair.split(":", 1) for pair in query) except ValueError: raise ValidationError( - {'query': _('Must be {field}:{value} pair(s)')}) + {'query': _('Must be : pair(s)')}) else: log.warning('Use of the "fields" parameter in resource_search is ' From b094c4eda6e61cacd3da3e638573803457f014d4 Mon Sep 17 00:00:00 2001 From: Toby Date: Tue, 3 Jul 2012 11:18:48 +0100 Subject: [PATCH 089/278] [#2362] toolkit docs --- doc/index.rst | 1 + doc/toolkit.rst | 148 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 doc/toolkit.rst diff --git a/doc/index.rst b/doc/index.rst index e8973a28272..ccd7a8d6dae 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -36,6 +36,7 @@ Customizing and Extending commenting extensions writing-extensions + toolkit forms tag-vocabularies form-integration diff --git a/doc/toolkit.rst b/doc/toolkit.rst new file mode 100644 index 00000000000..29ba543ab00 --- /dev/null +++ b/doc/toolkit.rst @@ -0,0 +1,148 @@ + +Plugins Toolkit +=============== + +To allow a safe way for extensions to interact with ckan a toolkit is +provided. We aim to keep this toolkit stable across ckan versions so +that extensions will work across diferent versions of ckan. + +.. Note:: + + It is advised that when writing extensions that all interaction with + ckan is done via the toolkit so that they do not break when new + versions of ckan are released. + +Over time we will be expanding the functionality available via +this toolkit. + +Example extension that registers a new helper function available to +templates via h.example_helper() :: + + import ckan.plugins as p + + + class ExampleExtension(p.SingletonPlugin): + + p.implements(p.IConfigurer) + p.implements(p.ITemplateHelpers) + + def update_config(self, config): + # add template directory that contains our snippet + p.toolkit.add_template_directory(config, 'templates') + + @classmethod + def example_helper(cls, data=None): + # render our custom snippet + return p.toolkit.render_snippet('custom_snippet.html', data) + + + def get_helpers(self): + # register our helper function + return {'example_helper': self.example_helper} + +The following functions, classes and exceptions are provided by the toolkit. + +*class* **CkanCommand** + Base class for building paster functions. + + +*exception* **CkanVersionException** + Exception raised if required ckan version is not available. + + +*exception* **NotAuthorized** + Exception raised when an action is not permitted by a user. + + +*exception* **ObjectNotFound** + Exception raised when an object cannot be found. + + +*exception* **ValidationError** + Exception raised when supplied data is invalid. + it contains details of the error that occurred. + + +**_** (*value*) + Mark a string for translation. Returns the localized unicode + string of value. + + Mark a string to be localized as follows:: + + _('This should be in lots of languages') + + + + +**add_public_directory** (*config, relative_path*) + Function to aid adding extra public paths to the config. + The path is relative to the file calling this function. + + +**add_template_directory** (*config, relative_path*) + Function to aid adding extra template paths to the config. + The path is relative to the file calling this function. + + +**asbool** (*obj*) + part of paste.deploy.converters: convert strings like yes, no, true, false, 0, 1 to boolean + + +**asint** (*obj*) + part of paste.deploy.converters: convert stings to integers + + +**aslist** (*obj, sep=None, strip=True*) + part of paste.deploy.converters: convert string objects to a list + + +**check_access** (*action, context, data_dict=None*) + check that the named action with the included context and + optional data dict is allowed raises NotAuthorized if the action is + not permitted or True. + + +**check_ckan_version** (*min_version=None, max_version=None*) + Check that the ckan version is correct for the plugin. + + +**get_action** (*action*) + Get the requested action function. + + +*class* **literal** + Represents an HTML literal. + + This subclass of unicode has a ``.__html__()`` method that is + detected by the ``escape()`` function. + + Also, if you add another string to this string, the other string + will be quoted and you will get back another literal object. Also + ``literal(...) % obj`` will quote any value(s) from ``obj``. If + you do something like ``literal(...) + literal(...)``, neither + string will be changed because ``escape(literal(...))`` doesn't + change the original literal. + + Changed in WebHelpers 1.2: the implementation is now now a subclass of + ``markupsafe.Markup``. This brings some new methods: ``.escape`` (class + method), ``.unescape``, and ``.striptags``. + + + + +**render** (*template_name, data=None*) + Main template render function. + + +**render_snippet** (*template_name, data=None*) + helper for the render_snippet function + similar to the render function. + + +**render_text** (*template_name, data=None*) + Render genshi text template. + + +**requires_ckan_version** (*min_version, max_version=None*) + Check that the ckan version is correct for the plugin. + From 3e00ffea420bd11af64b0d2d417e3c6100b71390 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:05:58 +0100 Subject: [PATCH 090/278] Make synchronous search the default behaviour Unless already loaded or explicitly disabled via the `ckan.search.automatic_index` configuration option, the synchronous search plugin will be loaded during startup time. --- ckan/config/deployment.ini_tmpl | 7 ++++++- ckan/config/environment.py | 8 ++++++++ doc/configuration.rst | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d3c0199687f..d66d6543248 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -24,7 +24,7 @@ app_instance_uuid = ${app_instance_uuid} # List the names of CKAN extensions to activate. # Note: This line is required to be here for packaging, even if it is empty. -ckan.plugins = stats synchronous_search +ckan.plugins = stats # If you'd like to fine-tune the individual locations of the cache data dirs # for the Cache data, or the Session saves, un-comment the desired settings @@ -112,6 +112,11 @@ ckan.gravatar_default = identicon ## Solr support #solr_url = http://127.0.0.1:8983/solr +## Automatic indexing. Make all changes immediately available via the search +## after editing or creating a dataset. Default is true. If for some reason +## you need the indexing to occur asynchronously, set this option to 0. +# ckan.search.automatic_indexing = 1 + ## An 'id' for the site (using, for example, when creating entries in a common search index) ## If not specified derived from the site_url # ckan.site_id = ckan.net diff --git a/ckan/config/environment.py b/ckan/config/environment.py index bb46ea2db8a..8fca75b7548 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -19,6 +19,7 @@ import ckan.lib.search as search import ckan.lib.app_globals as app_globals +log = logging.getLogger(__name__) # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) @@ -122,6 +123,13 @@ def find_controller(self, controller): # load all CKAN plugins p.load_all(config) + # Load the synchronous search plugin, unless already loaded or + # explicitly disabled + if not 'synchronous_search' in config.get('ckan.plugins') and \ + asbool(config.get('ckan.search.automatic_indexing',True)): + log.debug('Loading the synchronous search plugin') + p.load('synchronous_search') + for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) diff --git a/doc/configuration.rst b/doc/configuration.rst index 9b753e480d9..ebb09845bdc 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -496,6 +496,23 @@ Optionally, ``solr_user`` and ``solr_password`` can also be configured to specif Note, if you change this value, you need to rebuild the search index. +.. index:: + single: ckan.search.automatic_indexing + +ckan.search.automatic_indexing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.search.automatic_indexing = 1 + +Make all changes immediately available via the search after editing or +creating a dataset. Default is true. If for some reason you need the indexing +to occur asynchronously, set this option to 0. + +Note, this is equivalent to explicitly load the `synchronous_search` plugin. + + simple_search ^^^^^^^^^^^^^ From 08cba3144db70784f90c381a2139c8c035f2b63e Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:44:37 +0100 Subject: [PATCH 091/278] PEP8 environment.py --- ckan/config/environment.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 8fca75b7548..d388937ff33 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -24,6 +24,7 @@ # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) + class _Helpers(object): ''' Helper object giving access to template helpers stopping missing functions from causing template exceptions. Useful if @@ -94,13 +95,16 @@ def load_environment(global_conf, app_conf): from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = PylonsApp.find_controller + # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: - mycontroller = pkg_resources.EntryPoint.parse('x=%s' % controller).load(False) + mycontroller = pkg_resources \ + .EntryPoint \ + .parse('x=%s' % controller).load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) @@ -126,7 +130,7 @@ def find_controller(self, controller): # Load the synchronous search plugin, unless already loaded or # explicitly disabled if not 'synchronous_search' in config.get('ckan.plugins') and \ - asbool(config.get('ckan.search.automatic_indexing',True)): + asbool(config.get('ckan.search.automatic_indexing', True)): log.debug('Loading the synchronous search plugin') p.load('synchronous_search') @@ -160,11 +164,13 @@ def find_controller(self, controller): config['pylons.app_globals'] = app_globals.Globals() # add helper functions - restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'true')) + restrict_helpers = asbool( + config.get('ckan.restrict_template_vars', 'true')) helpers = _Helpers(h, restrict_helpers) config['pylons.h'] = helpers - ## redo template setup to use genshi.search_path (so remove std template setup) + # Redo template setup to use genshi.search_path + # (so remove std template setup) template_paths = [paths['templates'][0]] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: @@ -173,6 +179,7 @@ def find_controller(self, controller): # Translator (i18n) translator = Translator(pylons.translator) + def template_loaded(template): translator.setup(template) @@ -203,8 +210,6 @@ def template_loaded(template): # # ################################################################# - - ''' This code is based on Genshi code @@ -273,11 +278,14 @@ def genshi_lookup_attr(cls, obj, key): # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings - warnings.filterwarnings('ignore', '^Unicode type received non-unicode bind param value', sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning) + msgs = ['^Unicode type received non-unicode bind param value', + "^Did not recognize type 'BIGINT' of column 'size'", + "^Did not recognize type 'tsvector' of column 'search_vector'" + ] + for msg in msgs: + warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) - ckan_db = os.environ.get('CKAN_DB') + ckan_db = os.environ.get('CKAN_DB') if ckan_db: config['sqlalchemy.url'] = ckan_db @@ -296,4 +304,3 @@ def genshi_lookup_attr(cls, obj, key): for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) - From 63d82ceaea33277b21be3d336f1cd29d98959704 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 18:53:24 +0200 Subject: [PATCH 092/278] Update several dependencies Replaced PyUtilib==4.0.2848 with pyutilib.component.core==4.5.3, this avoids pulling in all the pyutilib packages and the packages they depend on. Updated webhelpers, solrpy, formalchemy, markupsafe, tempita, routes and paste to the latest versions from pypi --- pip-requirements.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 8a9fc42ce87..98f4a9d7150 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -5,23 +5,23 @@ Genshi==0.6 sqlalchemy-migrate==0.7.1 sqlalchemy==0.7.3 -webhelpers==1.2 -PyUtilib==4.0.2848 +webhelpers==1.3 +pyutilib.component.core==4.5.3 -e git+https://github.com/okfn/vdm.git@vdm-0.11#egg=vdm -solrpy==0.9.4 -formalchemy==1.4.1 +solrpy==0.9.5 +formalchemy==1.4.2 pairtree==0.7.1-T ofs==0.4.1 apachemiddleware==0.1.1 -markupsafe==0.9.2 +markupsafe==0.15 babel==0.9.4 psycopg2==2.0.13 webob==1.0.8 Pylons==0.9.7 repoze.who==1.0.19 -tempita==0.4 +tempita==0.5.1 zope.interface==3.5.3 repoze.who.plugins.openid==0.5.3 repoze.who-friendlyform==1.0.8 -routes==1.12 -paste==1.7.2 +routes==1.13 +paste==1.7.5.1 From a8dab115f0e94b7f475d15ffd6897bd03a6f81b4 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 28 Jun 2012 19:23:03 +0200 Subject: [PATCH 093/278] Update babel, psyco and zope dependencies --- pip-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 98f4a9d7150..3156743d773 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -14,13 +14,13 @@ pairtree==0.7.1-T ofs==0.4.1 apachemiddleware==0.1.1 markupsafe==0.15 -babel==0.9.4 -psycopg2==2.0.13 +babel==0.9.6 +psycopg2==2.4.5 webob==1.0.8 Pylons==0.9.7 repoze.who==1.0.19 tempita==0.5.1 -zope.interface==3.5.3 +zope.interface==4.0.1 repoze.who.plugins.openid==0.5.3 repoze.who-friendlyform==1.0.8 routes==1.13 From 755431512cf806b3af77a3942b613da369f719f8 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 29 Jun 2012 06:18:03 +0000 Subject: [PATCH 094/278] Upgrade sqlalchemy-migrate to 0.7.2 (latest) --- pip-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 3156743d773..5db1fc4c09a 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -3,7 +3,7 @@ # like: pip install -r pip-requirements.txt. See the Install from Source # instructions in CKAN's documentation for full installation instructions. Genshi==0.6 -sqlalchemy-migrate==0.7.1 +sqlalchemy-migrate==0.7.2 sqlalchemy==0.7.3 webhelpers==1.3 pyutilib.component.core==4.5.3 From 5629aade36916b967ffae250812e833816e0dddd Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 29 Jun 2012 06:18:42 +0000 Subject: [PATCH 095/278] Upgrade sqlalchemy to 0.7.8 (latest) --- pip-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip-requirements.txt b/pip-requirements.txt index 5db1fc4c09a..2bb53343f1a 100644 --- a/pip-requirements.txt +++ b/pip-requirements.txt @@ -4,7 +4,7 @@ # instructions in CKAN's documentation for full installation instructions. Genshi==0.6 sqlalchemy-migrate==0.7.2 -sqlalchemy==0.7.3 +sqlalchemy==0.7.8 webhelpers==1.3 pyutilib.component.core==4.5.3 -e git+https://github.com/okfn/vdm.git@vdm-0.11#egg=vdm From 9b72e88919fcdd25c0156a430f00968c9ec76ec3 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:02:30 +0200 Subject: [PATCH 096/278] [#2627] Refactor form_to_db_schema_options Make it call self.form_to_db_schema so that the form_to_db_schema() methods of IDatasetForm extensions get called. Also make it call new form_to_db_schema_api_create() and form_to_db_schema_api_update() methods which could potentially be overridden by extensions also. --- ckan/lib/plugins.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 258e1449681..4ce1f4236ca 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -221,11 +221,21 @@ def form_to_db_schema_options(self, options): if options.get('api'): if options.get('type') == 'create': - return logic.schema.default_create_package_schema() + return self.form_to_db_schema_api_create() else: - return logic.schema.default_update_package_schema() + assert options.get('type') == 'update' + return self.form_to_db_schema_api_update() else: - return logic.schema.package_form_schema() + return self.form_to_db_schema() + + def form_to_db_schema(self, options): + return logic.schema.package_form_schema() + + def form_to_db_schema_api_create(self): + return logic.schema.default_create_package_schema() + + def form_to_db_schema_api_update(self): + return logic.schema.default_update_package_schema() def db_to_form_schema(self): '''This is an interface to manipulate data from the database From d93cb58049bf79a1983f3b96759361c0627684a1 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:32:00 +0200 Subject: [PATCH 097/278] Fix a crasher in package/read.html template Sometimes c.pkg_dict has no member named groups (e.g. when there is an active IDatasetForm plugin with a db_to_form_schema() method). --- ckan/templates/package/read.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/read.html b/ckan/templates/package/read.html index 29a03720bbd..7c76ef39192 100644 --- a/ckan/templates/package/read.html +++ b/ckan/templates/package/read.html @@ -45,7 +45,7 @@

    Tags

    ${tag_list(c.pkg_dict.get('tags', ''))} -
  • +

  • Groups

    • From 6f91aa595b74e3ff4af2128cef11f76786f8ef20 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:51:11 +0200 Subject: [PATCH 098/278] Add db_to_form_schema method For IDatasetForm plugins to base their db_to_form_schema() dicts on --- ckan/logic/schema.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 8257408bb68..301f46b0b8b 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,6 +175,23 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema +def db_to_form_schema(): + schema = default_package_schema() + # Workaround a bug in CKAN's convert_from_tags() function. + # TODO: Fix this issue in convert_from_tags(). + schema.update({ + 'tags': { + '__extras': [ckan.lib.navl.validators.keep_extras, + ckan.logic.converters.free_tags_only] + }, + }) + # Workaround a bug in CKAN. + # TODO: Fix this elsewhere so we don't need to workaround it here. + schema['resources'].update({ + 'created': [ckan.lib.navl.validators.ignore_missing], + }) + return schema + def default_group_schema(): schema = { From 126c94b730b9c399dc2ba9c1a5c8858d7d97b802 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:02:05 +0200 Subject: [PATCH 099/278] Rename db_to_form_schema -> db_to_form_package_schema() --- ckan/logic/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 301f46b0b8b..5ae86675633 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,7 +175,7 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema -def db_to_form_schema(): +def db_to_form_package_schema(): schema = default_package_schema() # Workaround a bug in CKAN's convert_from_tags() function. # TODO: Fix this issue in convert_from_tags(). From 786479e8e6eea392fc4a4312c7c8a534a08083a8 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:20:14 +0200 Subject: [PATCH 100/278] Rename package_form_schema() -> form_to_db_package_schema() The only purpose of this schema is to be used as a base schema by IDatasetForm plugins for their form_to_db_schema() methods, hopefully the new name makes that clearer --- ckan/lib/plugins.py | 6 +++--- ckan/logic/schema.py | 2 +- ckan/tests/schema/test_schema.py | 2 +- ckanext/publisher_form/forms.py | 1 - ckanext/test_tag_vocab_plugin.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 4ce1f4236ca..bb5b795aa18 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -228,8 +228,8 @@ def form_to_db_schema_options(self, options): else: return self.form_to_db_schema() - def form_to_db_schema(self, options): - return logic.schema.package_form_schema() + def form_to_db_schema(self): + return logic.schema.form_to_db_package_schema() def form_to_db_schema_api_create(self): return logic.schema.default_create_package_schema() @@ -397,7 +397,7 @@ def check_data_dict(self, data_dict): 'extras_validation', 'save', 'return_to', 'resources'] - schema_keys = package_form_schema().keys() + schema_keys = form_to_db_package_schema().keys() keys_in_schema = set(schema_keys) - set(surplus_keys_schema) missing_keys = keys_in_schema - set(data_dict.keys()) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 5ae86675633..ba4f9472afb 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,7 +152,7 @@ def default_update_package_schema(): return schema -def package_form_schema(): +def form_to_db_package_schema(): schema = default_package_schema() ##new diff --git a/ckan/tests/schema/test_schema.py b/ckan/tests/schema/test_schema.py index e8d67e2c547..ab2c3d629d6 100644 --- a/ckan/tests/schema/test_schema.py +++ b/ckan/tests/schema/test_schema.py @@ -94,7 +94,7 @@ def test_tag_string_parsing(self): # errors correctly. context = {'model': ckan.model, 'session': ckan.model.Session} - schema = ckan.logic.schema.package_form_schema() + schema = ckan.logic.schema.form_to_db_package_schema() # basic parsing of comma separated values tests = (('tag', ['tag'], []), diff --git a/ckanext/publisher_form/forms.py b/ckanext/publisher_form/forms.py index 672209f6b4a..0d23d7aed10 100644 --- a/ckanext/publisher_form/forms.py +++ b/ckanext/publisher_form/forms.py @@ -9,7 +9,6 @@ from ckan.logic import tuplize_dict, clean_dict, parse_params import ckan.logic.schema as default_schema from ckan.logic.schema import group_form_schema -from ckan.logic.schema import package_form_schema import ckan.logic.validators as val from ckan.lib.base import BaseController, render, c, model, abort, request from ckan.lib.base import redirect, _, config, h diff --git a/ckanext/test_tag_vocab_plugin.py b/ckanext/test_tag_vocab_plugin.py index 043234890b2..2fe95f4ed9d 100644 --- a/ckanext/test_tag_vocab_plugin.py +++ b/ckanext/test_tag_vocab_plugin.py @@ -7,7 +7,7 @@ from genshi.filters import Transformer from ckan.logic import get_action from ckan.logic.converters import convert_to_tags, convert_from_tags, free_tags_only -from ckan.logic.schema import package_form_schema, default_package_schema +from ckan.logic.schema import default_package_schema from ckan.lib.navl.validators import ignore_missing, keep_extras from ckan import plugins From 74b346ee9a2bbbc48021ffb2e3ead6fbd5923654 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:32:51 +0200 Subject: [PATCH 101/278] Add back in deprectated package_form_schema() for backwards compatibility --- ckan/logic/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index ba4f9472afb..2bb0b58b3ac 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,6 +152,11 @@ def default_update_package_schema(): return schema +def package_form_schema(): + # This function is deprecated and was replaced by + # form_to_db_package_schema(), it remains here for backwards compatibility. + return form_to_db_package_schema() + def form_to_db_package_schema(): schema = default_package_schema() From cd7974b2be434d88d937f1fbad2042413b463c48 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:02:30 +0200 Subject: [PATCH 102/278] Refactor form_to_db_schema_options Make it call self.form_to_db_schema so that the form_to_db_schema() methods of IDatasetForm extensions get called. Also make it call new form_to_db_schema_api_create() and form_to_db_schema_api_update() methods which could potentially be overridden by extensions also. --- ckan/lib/plugins.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 258e1449681..4ce1f4236ca 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -221,11 +221,21 @@ def form_to_db_schema_options(self, options): if options.get('api'): if options.get('type') == 'create': - return logic.schema.default_create_package_schema() + return self.form_to_db_schema_api_create() else: - return logic.schema.default_update_package_schema() + assert options.get('type') == 'update' + return self.form_to_db_schema_api_update() else: - return logic.schema.package_form_schema() + return self.form_to_db_schema() + + def form_to_db_schema(self, options): + return logic.schema.package_form_schema() + + def form_to_db_schema_api_create(self): + return logic.schema.default_create_package_schema() + + def form_to_db_schema_api_update(self): + return logic.schema.default_update_package_schema() def db_to_form_schema(self): '''This is an interface to manipulate data from the database From 90012807433047593f7acac22522af5bc1302ac6 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:32:00 +0200 Subject: [PATCH 103/278] Fix a crasher in package/read.html template Sometimes c.pkg_dict has no member named groups (e.g. when there is an active IDatasetForm plugin with a db_to_form_schema() method). --- ckan/templates/package/read.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/read.html b/ckan/templates/package/read.html index 29a03720bbd..7c76ef39192 100644 --- a/ckan/templates/package/read.html +++ b/ckan/templates/package/read.html @@ -45,7 +45,7 @@

      Tags

      ${tag_list(c.pkg_dict.get('tags', ''))}
      • -
    • +

    • Groups

      • From a5bedce1d321752fd083a7a4ec10e263439e713d Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:51:11 +0200 Subject: [PATCH 104/278] Add db_to_form_schema method For IDatasetForm plugins to base their db_to_form_schema() dicts on --- ckan/logic/schema.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 8257408bb68..301f46b0b8b 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,6 +175,23 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema +def db_to_form_schema(): + schema = default_package_schema() + # Workaround a bug in CKAN's convert_from_tags() function. + # TODO: Fix this issue in convert_from_tags(). + schema.update({ + 'tags': { + '__extras': [ckan.lib.navl.validators.keep_extras, + ckan.logic.converters.free_tags_only] + }, + }) + # Workaround a bug in CKAN. + # TODO: Fix this elsewhere so we don't need to workaround it here. + schema['resources'].update({ + 'created': [ckan.lib.navl.validators.ignore_missing], + }) + return schema + def default_group_schema(): schema = { From 7f59f2539abfa009618d34deea087a97a4a94dba Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:02:05 +0200 Subject: [PATCH 105/278] Rename db_to_form_schema -> db_to_form_package_schema() --- ckan/logic/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 301f46b0b8b..5ae86675633 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,7 +175,7 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema -def db_to_form_schema(): +def db_to_form_package_schema(): schema = default_package_schema() # Workaround a bug in CKAN's convert_from_tags() function. # TODO: Fix this issue in convert_from_tags(). From 38fc15cfc0965e567444b2653ff5a36b3f33828b Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:20:14 +0200 Subject: [PATCH 106/278] Rename package_form_schema() -> form_to_db_package_schema() The only purpose of this schema is to be used as a base schema by IDatasetForm plugins for their form_to_db_schema() methods, hopefully the new name makes that clearer --- ckan/lib/plugins.py | 6 +++--- ckan/logic/schema.py | 2 +- ckan/tests/schema/test_schema.py | 2 +- ckanext/publisher_form/forms.py | 1 - ckanext/test_tag_vocab_plugin.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 4ce1f4236ca..bb5b795aa18 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -228,8 +228,8 @@ def form_to_db_schema_options(self, options): else: return self.form_to_db_schema() - def form_to_db_schema(self, options): - return logic.schema.package_form_schema() + def form_to_db_schema(self): + return logic.schema.form_to_db_package_schema() def form_to_db_schema_api_create(self): return logic.schema.default_create_package_schema() @@ -397,7 +397,7 @@ def check_data_dict(self, data_dict): 'extras_validation', 'save', 'return_to', 'resources'] - schema_keys = package_form_schema().keys() + schema_keys = form_to_db_package_schema().keys() keys_in_schema = set(schema_keys) - set(surplus_keys_schema) missing_keys = keys_in_schema - set(data_dict.keys()) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 5ae86675633..ba4f9472afb 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,7 +152,7 @@ def default_update_package_schema(): return schema -def package_form_schema(): +def form_to_db_package_schema(): schema = default_package_schema() ##new diff --git a/ckan/tests/schema/test_schema.py b/ckan/tests/schema/test_schema.py index e8d67e2c547..ab2c3d629d6 100644 --- a/ckan/tests/schema/test_schema.py +++ b/ckan/tests/schema/test_schema.py @@ -94,7 +94,7 @@ def test_tag_string_parsing(self): # errors correctly. context = {'model': ckan.model, 'session': ckan.model.Session} - schema = ckan.logic.schema.package_form_schema() + schema = ckan.logic.schema.form_to_db_package_schema() # basic parsing of comma separated values tests = (('tag', ['tag'], []), diff --git a/ckanext/publisher_form/forms.py b/ckanext/publisher_form/forms.py index 672209f6b4a..0d23d7aed10 100644 --- a/ckanext/publisher_form/forms.py +++ b/ckanext/publisher_form/forms.py @@ -9,7 +9,6 @@ from ckan.logic import tuplize_dict, clean_dict, parse_params import ckan.logic.schema as default_schema from ckan.logic.schema import group_form_schema -from ckan.logic.schema import package_form_schema import ckan.logic.validators as val from ckan.lib.base import BaseController, render, c, model, abort, request from ckan.lib.base import redirect, _, config, h diff --git a/ckanext/test_tag_vocab_plugin.py b/ckanext/test_tag_vocab_plugin.py index 043234890b2..2fe95f4ed9d 100644 --- a/ckanext/test_tag_vocab_plugin.py +++ b/ckanext/test_tag_vocab_plugin.py @@ -7,7 +7,7 @@ from genshi.filters import Transformer from ckan.logic import get_action from ckan.logic.converters import convert_to_tags, convert_from_tags, free_tags_only -from ckan.logic.schema import package_form_schema, default_package_schema +from ckan.logic.schema import default_package_schema from ckan.lib.navl.validators import ignore_missing, keep_extras from ckan import plugins From c5553b8e0fdd3e0394c97880f343bee4615b2301 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:32:51 +0200 Subject: [PATCH 107/278] Add back in deprectated package_form_schema() for backwards compatibility --- ckan/logic/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index ba4f9472afb..2bb0b58b3ac 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,6 +152,11 @@ def default_update_package_schema(): return schema +def package_form_schema(): + # This function is deprecated and was replaced by + # form_to_db_package_schema(), it remains here for backwards compatibility. + return form_to_db_package_schema() + def form_to_db_package_schema(): schema = default_package_schema() From efca8cb18ca6379a51cb81e59d347ecb370ec78c Mon Sep 17 00:00:00 2001 From: kindly Date: Fri, 6 Jul 2012 15:01:45 +0100 Subject: [PATCH 108/278] [2627] fix some more date issues when displayed --- ckan/logic/schema.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 2bb0b58b3ac..f3b27654b74 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -194,6 +194,9 @@ def db_to_form_package_schema(): # TODO: Fix this elsewhere so we don't need to workaround it here. schema['resources'].update({ 'created': [ckan.lib.navl.validators.ignore_missing], + 'last_modified': [ckan.lib.navl.validators.ignore_missing], + 'cache_last_updated': [ckan.lib.navl.validators.ignore_missing], + 'webstore_last_updated': [ckan.lib.navl.validators.ignore_missing], }) return schema From 519472c1bcf15f66e18927f775e06e5ee70724fa Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 12:21:55 +0100 Subject: [PATCH 109/278] [#2437] Add config file argument to paster commands. --- doc/coding-standards.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 5f575c174b0..9ed4e291b04 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -389,7 +389,7 @@ CKAN currently uses to manage these scripts. When you deploy new code to a CKAN instance, as part of the process you run any required migration scripts with: :: - paster db upgrade + paster --plugin=ckan db upgrade --config={.ini file} The scripts give their model version numbers in their filenames and are stored in ``ckan/migration/versions/``. @@ -428,14 +428,16 @@ made: 1. Get a dump of the database schema before you add your new migrate scripts. :: - paster db clean - paster db upgrade + paster --plugin=ckan db clean --config={.ini file} + paster --plugin=ckan db upgrade --config={.ini file} pg_dump -h host -s -f old.sql dbname 2. Get a dump of the database as you have specified it in the model. :: - paster db clean - paster db create-from-model #this makes the database as defined in the model + paster --plugin=ckan db clean --config={.ini file} + + #this makes the database as defined in the model + paster --plugin=ckan db create-from-model -config={.ini file} pg_dump -h host -s -f new.sql dbname 3. Get agpdiff (apt-get it). It produces sql it thinks that you need to run on From 29a06dc503681cb0ab7951ea05cf55b47d025998 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 12:28:25 +0100 Subject: [PATCH 110/278] [#2437] Update commit message guidelines --- doc/coding-standards.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 9ed4e291b04..32987bc6150 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -66,9 +66,10 @@ Feature Branches ---------------- All ticketed work should be developed on a corresponding feature branch forked -from master. The name of the branch should inlude the `type` of the ticket, -the ticket's number, and a brief one-line synopsis of the purpose of the -ticket. eg: ``enhancement-2298-add-sort-by-controls-to-search-page``. +from master. The name of the branch should inlude the ticket's number, the +ticket type, and a brief one-line synopsis of the purpose of the ticket. eg: +``2298-feature-add-sort-by-controls-to-search-page``. This allows the ticket +number to be esaily searchable through github's web interface. Once work on the branch has been completed and it is ready to be merged into master, make a pull request on github. Another member of the CKAN team will From 9063f7be8bf73f5a85ee527838c27056facce6a4 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 13:43:59 +0100 Subject: [PATCH 111/278] [#2313] Remove deprecated facet_items helper. --- ckan/lib/helpers.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 114efb0a78e..0063c9618df 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -377,34 +377,6 @@ def unselected_facet_items(facet, limit=10): facets.append(facet_item) return sorted(facets, key=lambda item: item['count'], reverse=True)[:limit] -@deprecated() -def facet_items(*args, **kwargs): - """ - DEPRECATED: Use the new facet data structure, and `unselected_facet_items()` - """ - _log.warning('Deprecated function: ckan.lib.helpers:facet_items(). Will be removed in v1.8') - # facet_items() used to need c passing as the first arg - # this is deprecated as pointless - # throws error if ckan.restrict_template_vars is True - # When we move to strict helpers then this should be removed as a wrapper - if len(args) > 2 or (len(args) > 0 and 'name' in kwargs) or (len(args) > 1 and 'limit' in kwargs): - if not asbool(config.get('ckan.restrict_template_vars', 'false')): - return _facet_items(*args[1:], **kwargs) - raise Exception('facet_items() calling has been changed. remove c in template %s or included one' % c.__template_name) - return _facet_items(*args, **kwargs) - - -def _facet_items(name, limit=10): - if not c.facets or not c.facets.get(name): - return [] - facets = [] - for k, v in c.facets.get(name).items(): - if not len(k.strip()): - continue - if not (name, k) in request.params.items(): - facets.append((k, v)) - return sorted(facets, key=lambda (k, v): v, reverse=True)[:limit] - def facet_title(name): return config.get('search.facets.%s.title' % name, name.capitalize()) From 04a9fd7be98675eeb472e6695234c36cb2d6674a Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 13:55:12 +0100 Subject: [PATCH 112/278] [#2313] Removed deprecated facet_sidebar() function --- ckan/templates/facets.html | 54 ------------------- .../templates/publisher_read.html | 4 +- 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/ckan/templates/facets.html b/ckan/templates/facets.html index 001be8c79c3..939f3ee281b 100644 --- a/ckan/templates/facets.html +++ b/ckan/templates/facets.html @@ -75,60 +75,6 @@

        ${h.facet_title(title)}

        • - - - -
        -

        ${title(code)}

        - -

        ${if_empty}

        -
        -         - - - - -
      • ${if_empty}
        • -
      • - ${label(facet_item.name)} (facet_item['count']) -
        • -         -
        From aaff2a1175a4ecb16c272378c8facfedd6a56412 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 15:03:37 +0100 Subject: [PATCH 114/278] [#2313] Remove c.facets. It's been deprecated by c.search_facets. --- ckan/controllers/group.py | 2 -- ckan/controllers/home.py | 7 ++++--- ckan/controllers/package.py | 2 -- ckan/logic/action/get.py | 3 +-- ckan/tests/logic/test_action.py | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 8bb6119f513..41bec6ee26f 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -200,13 +200,11 @@ def pager_url(q=None, page=None): item_count=query['count'], items_per_page=limit ) - c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True - c.facets = {} c.page = h.Page(collection=[]) # Add the group's activity stream (already rendered to HTML) to the diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index e885f3bda9e..79e1d1ec889 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -49,13 +49,14 @@ def index(self): query = ckan.logic.get_action('package_search')( context, data_dict) c.package_count = query['count'] - c.facets = query['facets'] + c.search_facets = query['search_facets'] data_dict = {'order_by': 'packages', 'all_fields': 1} # only give the terms to group dictize that are returned in the # facets as full results take a lot longer - if 'groups' in c.facets: - data_dict['groups'] = c.facets['groups'].keys() + if 'groups' in c.search_facets: + data_dict['groups'] = [ item['name'] for item in + c.search_facets['groups']['items'] ] c.groups = ckan.logic.get_action('group_list')(context, data_dict) except SearchError, se: c.package_count = 0 diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index ccab6931135..218c746f7db 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -216,13 +216,11 @@ def pager_url(q=None, page=None): item_count=query['count'], items_per_page=limit ) - c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True - c.facets = {} c.page = h.Page(collection=[]) return render(self._search_template(package_type)) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index acdab602e75..460aca6f3b7 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1137,13 +1137,12 @@ def package_search(context, data_dict): search_results = { 'count': count, - 'facets': facets, 'results': results } # Transform facets into a more useful data structure. restructured_facets = {} - for key, value in search_results['facets'].items(): + for key, value in facets.items(): restructured_facets[key] = { 'title': key, 'items': [] diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 3b3566fecf7..3aa97bd91f9 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1318,7 +1318,7 @@ def after_search(self, search_results, search_params): assert 'results' in search_results assert 'count' in search_results - assert 'facets' in search_results + assert 'search_facets' in search_results if 'extras' in search_params and 'ext_avoid' in search_params['extras']: # Remove results with a certain value From 73f839b31b0c17e97190c0ab01ae72b86d473249 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sun, 8 Jul 2012 10:25:17 +0100 Subject: [PATCH 115/278] [#2313] Add changes to changelog --- CHANGELOG.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3a117515ddf..ad9c5dd9feb 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -24,6 +24,12 @@ v1.8 API has been rewritten, with each function and its arguments and return values now individually documented. +API changes and deprecation: +* [#2313] Deprecated functions related to the old faceting data structure have + been removed. `helpers.py:facet_items()`, `facets.html:facet_sidebar()`, + `facets.html:facet_list_items()` and the context is no longer populated with + `facets` field, use `search_facets` instead. + v1.7.1 2012-06-20 ================= From 3a518547e18a1b9d2177db9243620272c58f3ba9 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 9 Jul 2012 09:43:38 +0100 Subject: [PATCH 116/278] Fixes 2624, where tags were not saved with organization_dataset Added tag_convert_string to the validators for the form_to_db_schema so that it correctly converted the string to tags in the DB. --- ckanext/organizations/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/organizations/forms.py b/ckanext/organizations/forms.py index caf9226c2da..3e22ff4e297 100644 --- a/ckanext/organizations/forms.py +++ b/ckanext/organizations/forms.py @@ -11,7 +11,7 @@ from ckan.plugins import IGroupForm, IDatasetForm, IConfigurer, IRoutes from ckan.plugins import implements, SingletonPlugin from ckan.logic import check_access - +from ckan.logic.validators import tag_string_convert from ckan.lib.navl.validators import (ignore_missing, not_empty, empty, @@ -237,6 +237,7 @@ def form_to_db_schema(self): schema = default_package_schema() schema['groups']['capacity'] = [ignore_missing, unicode] schema['__after'] = [group_required] + schema['tag_string'] = [ignore_missing, tag_string_convert] return schema def check_data_dict(self, data_dict, schema=None): From 4f32c46e50d2c9767faf6768ac013248d83d8d82 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Jul 2012 12:37:32 +0200 Subject: [PATCH 117/278] [#2624] Base organizations form_to_db_schema() off form_to_db_package_schema() --- ckanext/organizations/forms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckanext/organizations/forms.py b/ckanext/organizations/forms.py index 3e22ff4e297..5e33eaa8f24 100644 --- a/ckanext/organizations/forms.py +++ b/ckanext/organizations/forms.py @@ -3,7 +3,7 @@ import ckan.authz as authz from ckan.logic import NotAuthorized -from ckan.logic.schema import group_form_schema, default_package_schema +from ckan.logic.schema import group_form_schema, form_to_db_package_schema from ckan.lib import base from ckan.lib.base import c, model, abort, request from ckan.lib.base import redirect, _, config, h @@ -234,10 +234,9 @@ def db_to_form_schema(self): into a format suitable for the form (optional)''' def form_to_db_schema(self): - schema = default_package_schema() + schema = form_to_db_package_schema() schema['groups']['capacity'] = [ignore_missing, unicode] schema['__after'] = [group_required] - schema['tag_string'] = [ignore_missing, tag_string_convert] return schema def check_data_dict(self, data_dict, schema=None): From 3bae84bd1352d4a2b9fe37579f7cfcf87fef49ab Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 9 Jul 2012 11:57:20 +0100 Subject: [PATCH 118/278] [#2634] add restricted template vars to change log --- CHANGELOG.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3a117515ddf..e1eabee6a85 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -23,6 +23,13 @@ v1.8 * [#2345] New action API reference docs. The documentation for CKAN's Action API has been rewritten, with each function and its arguments and return values now individually documented. +* Template helper functions are now restricted by default. This is + part of a process of refactoring the CKAN code. By default only + those helper functions listed in lib.helpers.__allowed_functions__ + are available to templates. The full functions can still be made + available by `setting ckan.restrict_template_vars = false` in your + .ini config file. Only restricted functions will be allowed in + future versions of CKAN. v1.7.1 2012-06-20 ================= From b41b57c5fb686c76a6a1a1dd20df5a358b3c7a93 Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 9 Jul 2012 12:45:27 +0100 Subject: [PATCH 119/278] [#2632] check allowed helper functions exist --- ckan/config/environment.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index d388937ff33..f07bb1d979a 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -32,7 +32,7 @@ class _Helpers(object): not been enabled. ''' def __init__(self, helpers, restrict=True): functions = {} - allowed = helpers.__allowed_functions__ + allowed = helpers.__allowed_functions__[:] # list of functions due to be deprecated self.deprecated = [] @@ -42,8 +42,13 @@ def __init__(self, helpers, restrict=True): if restrict: continue functions[helper] = getattr(helpers, helper) + allowed.remove(helper) self.functions = functions + if allowed: + raise Exception('Template helper function(s) `%s` not defined' + % ', '.join(allowed)) + # extend helper functions with ones supplied by plugins extra_helpers = [] for plugin in p.PluginImplementations(p.ITemplateHelpers): From 9a072d30f89429e3913a7f4bca237a19aea202c8 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 9 Jul 2012 13:16:07 +0100 Subject: [PATCH 120/278] Updated after dodgy merge --- ckan/logic/action/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index ff0c3fc9587..f37640eee8d 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1915,7 +1915,7 @@ def _render_follow_user_activity(context, activity): 'changed group' : _render_changed_group_activity, 'deleted group' : _render_deleted_group_activity, 'new related item': _render_new_related_activity, - 'deleted related item': _render_deleted_related_activity + 'deleted related item': _render_deleted_related_activity, 'follow dataset': _render_follow_dataset_activity, 'follow user': _render_follow_user_activity, } From cab9e7ddb8efdae373c97fc8523063b3e60d5320 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 9 Jul 2012 13:27:31 +0100 Subject: [PATCH 121/278] Make sure that featured item checkbox is checked when filtered --- ckan/templates/related/dashboard.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/related/dashboard.html b/ckan/templates/related/dashboard.html index 16ff7385238..64383c390a8 100644 --- a/ckan/templates/related/dashboard.html +++ b/ckan/templates/related/dashboard.html @@ -53,7 +53,7 @@

        ${c.page.item_count} related - +

        From f6da5124e79d5aad6c14b5e4451e5eae07fdac9e Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 Jul 2012 15:03:51 +0100 Subject: [PATCH 122/278] Change default authorization settings These are the new default authotization behaviours: * Only logged in users are able to create datasets and upload files * Only the user that created the dataset can edit it by default NB: this will break a large number of test. They will be fixed on following commits. --- ckan/config/deployment.ini_tmpl | 4 ++-- ckan/model/authz.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d3c0199687f..5d9b4e4826c 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -141,9 +141,9 @@ ckan.dump_dir = %(here)s/dump ckan.backup_dir = %(here)s/backup # Default authorizations for new domain objects -#ckan.default_roles.Package = {"visitor": ["editor"], "logged_in": ["editor"]} +#ckan.default_roles.Package = {"visitor": ["reader"], "logged_in": ["reader"]} #ckan.default_roles.Group = {"visitor": ["reader"], "logged_in": ["reader"]} -#ckan.default_roles.System = {"visitor": ["anon_editor"], "logged_in": ["editor"]} +#ckan.default_roles.System = {"visitor": ["reader"], "logged_in": ["editor"]} #ckan.default_roles.AuthorizationGroup = {"visitor": ["reader"], "logged_in": ["reader"]} ## Ckan public and private recaptcha keys [localhost] diff --git a/ckan/model/authz.py b/ckan/model/authz.py index c95af3e43a5..7c952760ba4 100644 --- a/ckan/model/authz.py +++ b/ckan/model/authz.py @@ -406,9 +406,9 @@ def give_all_packages_default_user_roles(): # default user roles - used when the config doesn\'t specify them default_default_user_roles = { - 'Package': {"visitor": ["editor"], "logged_in": ["editor"]}, + 'Package': {"visitor": ["reader"], "logged_in": ["reader"]}, 'Group': {"visitor": ["reader"], "logged_in": ["reader"]}, - 'System': {"visitor": ["anon_editor"], "logged_in": ["editor"]}, + 'System': {"visitor": ["reader"], "logged_in": ["editor"]}, 'AuthorizationGroup': {"visitor": ["reader"], "logged_in": ["reader"]}, } From 2b0932a105d9c1292814b2afac4bf237a8db3008 Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 9 Jul 2012 15:42:24 +0100 Subject: [PATCH 123/278] [#2638] get_request_param() helper function --- ckan/lib/helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 114efb0a78e..586355c0205 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -885,6 +885,11 @@ def dashboard_activity_stream(user_id): context = {'model' : model, 'session':model.Session, 'user':c.user} return logic.get_action('dashboard_activity_list_html')(context, {'id': user_id}) +def get_request_param(parameter_name, default=None): + ''' This function allows templates to access query string parameters + from the request. This is useful for things like sort order in + searches. ''' + return request.params.get(parameter_names, default) # these are the functions that will end up in `h` template helpers # if config option restrict_template_vars is true @@ -942,6 +947,7 @@ def dashboard_activity_stream(user_id): 'follow_button', 'follow_count', 'dashboard_activity_stream', + 'get_request_param', # imported into ckan.lib.helpers 'literal', 'link_to', From b55c85e4fbc975c4e46dad1c15151ede3f0a6b06 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 Jul 2012 16:37:55 +0100 Subject: [PATCH 124/278] Fix broken tests after default auth settings change All tests involving non logged-in editing were failing, as this is no longer allowed. The changes on auth settings basically mean than everytime some editing is tested, a valid user with permissions on the dataset must be provided, e.g.: offset = url_for(controller='package', action='edit', id=myid) self.app.get(offset,extra_environ={'REMOTE_USER': 'russianfan'}) or: res = self.app.post('/api/action/resource_create', params=postparams, extra_environ={'Authorization': api_key }) --- ckan/tests/forms/test_authz.py | 8 +- ckan/tests/functional/api/test_activity.py | 110 +++++++------ ckan/tests/functional/test_admin.py | 4 +- ckan/tests/functional/test_authz.py | 2 +- ckan/tests/functional/test_home.py | 3 +- ckan/tests/functional/test_package.py | 149 ++++++++---------- .../functional/test_package_edit_authz.py | 67 +++----- ckan/tests/functional/test_search.py | 7 - ckan/tests/functional/test_storage.py | 16 +- ckan/tests/functional/test_upload.py | 5 +- ckan/tests/logic/test_action.py | 7 +- ckan/tests/models/test_authz.py | 22 ++- ckan/tests/models/test_repo.py | 4 +- 13 files changed, 203 insertions(+), 201 deletions(-) diff --git a/ckan/tests/forms/test_authz.py b/ckan/tests/forms/test_authz.py index 3b02b00ed0d..f7d31c143db 100644 --- a/ckan/tests/forms/test_authz.py +++ b/ckan/tests/forms/test_authz.py @@ -47,11 +47,11 @@ def teardown_class(self): def test_0_no_change(self): pkg_name, user = self._new_pkg(0) - before_roles = { 'visitor': 'editor', - 'logged_in': 'editor', + before_roles = { 'visitor': 'reader', + 'logged_in': 'reader', user.name: 'admin' } - after_roles = { 'visitor': 'editor', - 'logged_in': 'editor', + after_roles = { 'visitor': 'reader', + 'logged_in': 'reader', user.name: 'admin' } num_prs_before = model.Session.query(model.PackageRole).filter_by(user_id=user.id).count() self._test_change(before_roles, after_roles, pkg_name) diff --git a/ckan/tests/functional/api/test_activity.py b/ckan/tests/functional/api/test_activity.py index 4662909ca88..da34969e60a 100644 --- a/ckan/tests/functional/api/test_activity.py +++ b/ckan/tests/functional/api/test_activity.py @@ -2,6 +2,8 @@ import logging logger = logging.getLogger(__name__) +from nose.plugins.skip import SkipTest + import ckan import ckan.model as model from ckan.logic.action.create import package_create as _package_create @@ -1302,21 +1304,25 @@ def test_01_update_extras_not_logged_in(self): when a package extra is changed by a user who is not logged in. """ - context = { - 'model': model, - 'session': model.Session, - 'user': self.normal_user.name, - 'extras_as_string': True, - } - packages_with_extras = [] - for package_name in package_list(context, {}): - package_dict = package_show(context, {'id': package_name}) - if len(package_dict['extras']) > 0: - packages_with_extras.append(package_dict) - assert len(packages_with_extras) > 0, ( - "Need some packages with extras to test") - for package_dict in packages_with_extras: - self._update_extra(package_dict, None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# context = { +# 'model': model, +# 'session': model.Session, +# 'user': self.normal_user.name, +# 'extras_as_string': True, +# } +# packages_with_extras = [] +# for package_name in package_list(context, {}): +# package_dict = package_show(context, {'id': package_name}) +# if len(package_dict['extras']) > 0: +# packages_with_extras.append(package_dict) +# assert len(packages_with_extras) > 0, ( +# "Need some packages with extras to test") +# for package_dict in packages_with_extras: +# self._update_extra(package_dict, None) +# def test_01_update_package(self): """ @@ -1337,8 +1343,11 @@ def test_01_update_package_not_logged_in(self): when packages are updated by a user who is not logged in. """ - for package in model.Session.query(model.Package).all(): - self._update_package(package, user=None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# for package in model.Session.query(model.Package).all(): +# self._update_package(package, user=None) def test_01_update_resource(self): """ @@ -1386,7 +1395,10 @@ def test_create_package_not_logged_in(self): when a new package is created by a user who is not logged in. """ - self._create_package(user=None, name="not_logged_in_test_package") + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# self._create_package(user=None, name="not_logged_in_test_package") def test_add_resources(self): """ @@ -1407,8 +1419,11 @@ def test_add_resources_not_logged_in(self): when a resource is added to a package by a user who is not logged in. """ - for package in model.Session.query(model.Package).all(): - self._add_resource(package, user=None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# for package in model.Session.query(model.Package).all(): +# self._add_resource(package, user=None) def test_delete_package(self): """ @@ -1970,16 +1985,19 @@ def test_add_extras_not_logged_in(self): when an extra is added to a package by a user who is not logged in. """ - context = { - 'model': model, - 'session': model.Session, - 'user': self.normal_user.name, - 'extras_as_string': True, - } - for package_name in package_list(context, {}): - package_dict = package_show(context, {'id': package_name}) - self._add_extra(package_dict, None, key='not_logged_in_extra_key') + raise SkipTest +## TODO: remove, non logged in users can no longer edit packages +# context = { +# 'model': model, +# 'session': model.Session, +# 'user': self.normal_user.name, +# 'extras_as_string': True, +# } +# for package_name in package_list(context, {}): +# package_dict = package_show(context, {'id': package_name}) +# self._add_extra(package_dict, None, key='not_logged_in_extra_key') +# def test_delete_extras(self): """ Test deleted package extra activity stream. @@ -2012,21 +2030,25 @@ def test_delete_extras_not_logged_in(self): when a package extra is deleted by a user who is not logged in. """ - context = { - 'model': model, - 'session': model.Session, - 'user': self.normal_user.name, - 'extras_as_string': True, - } - packages_with_extras = [] - for package_name in package_list(context, {}): - package_dict = package_show(context, {'id': package_name}) - if len(package_dict['extras']) > 0: - packages_with_extras.append(package_dict) - assert len(packages_with_extras) > 0, ( - "Need some packages with extras to test") - for package_dict in packages_with_extras: - self._delete_extra(package_dict, None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# context = { +# 'model': model, +# 'session': model.Session, +# 'user': self.normal_user.name, +# 'extras_as_string': True, +# } +# packages_with_extras = [] +# for package_name in package_list(context, {}): +# package_dict = package_show(context, {'id': package_name}) +# if len(package_dict['extras']) > 0: +# packages_with_extras.append(package_dict) +# assert len(packages_with_extras) > 0, ( +# "Need some packages with extras to test") +# for package_dict in packages_with_extras: +# self._delete_extra(package_dict, None) +# def test_follow_dataset(self): user = self.normal_user diff --git a/ckan/tests/functional/test_admin.py b/ckan/tests/functional/test_admin.py index 416f1b0b548..ad5bf3f21cb 100644 --- a/ckan/tests/functional/test_admin.py +++ b/ckan/tests/functional/test_admin.py @@ -109,7 +109,7 @@ def authz_submit(form): # before we start changing things, check that the roles on the system are as expected assert original_user_roles == \ - [(u'logged_in', u'editor'), (u'testsysadmin', u'admin'), (u'visitor', u'anon_editor')] , \ + [(u'logged_in', u'editor'), (u'testsysadmin', u'admin'), (u'visitor', u'reader')] , \ "original user roles not as expected " + str(original_user_roles) assert original_authzgroup_roles == [(u'anauthzgroup', u'editor')], \ @@ -160,7 +160,7 @@ def authz_submit(form): # change lots of things form = get_user_form() check_and_set_checkbox(form, u'visitor', u'editor', False, True) - check_and_set_checkbox(form, u'visitor', u'reader', False, False) + check_and_set_checkbox(form, u'visitor', u'reader', True, False) check_and_set_checkbox(form, u'logged_in', u'editor', True, False) check_and_set_checkbox(form, u'logged_in', u'reader', False, True) submit(form) diff --git a/ckan/tests/functional/test_authz.py b/ckan/tests/functional/test_authz.py index 6b2e55dd62b..bd8bcb9f11e 100644 --- a/ckan/tests/functional/test_authz.py +++ b/ckan/tests/functional/test_authz.py @@ -342,7 +342,7 @@ def test_02_visitor_edits(self): self._test_can('edit', self.visitor, [], interfaces=['rest']) def test_visitor_creates(self): - self._test_can('create', self.visitor, [], interfaces=['wui'], entity_types=['dataset']) + self._test_cant('create', self.visitor, [], interfaces=['wui'], entity_types=['dataset']) self._test_cant('create', self.visitor, [], interfaces=['wui'], entity_types=['group']) # need to be sysadmin self._test_cant('create', self.visitor, [], interfaces=['rest']) diff --git a/ckan/tests/functional/test_home.py b/ckan/tests/functional/test_home.py index f47893dbb48..f69ca90d5da 100644 --- a/ckan/tests/functional/test_home.py +++ b/ckan/tests/functional/test_home.py @@ -25,7 +25,8 @@ def teardown_class(self): def test_home_page(self): offset = url_for('home') res = self.app.get(offset) - assert 'Add a dataset' in res + # Non logged in users can no longer add datasets + assert not 'Add a dataset' in res assert 'Could not change language' not in res assert "Dave's books has 2 datasets" in res, res assert "Roger's books has 1 datasets" in res, res diff --git a/ckan/tests/functional/test_package.py b/ckan/tests/functional/test_package.py index ffe423737fb..59911a2bc06 100644 --- a/ckan/tests/functional/test_package.py +++ b/ckan/tests/functional/test_package.py @@ -199,7 +199,7 @@ def check_form_filled_correctly(self, res, **params): assert params['log_message'] in main_res, main_res def _check_redirect(self, return_url_param, expected_redirect, - pkg_name_to_edit=''): + pkg_name_to_edit='',extra_environ=None): ''' @param return_url_param - encoded url to be given as param - if None then assume redirect is specified in pylons config @@ -223,12 +223,12 @@ def _check_redirect(self, return_url_param, expected_redirect, if return_url_param: offset_params['return_to'] = return_url_param offset = url_for(**offset_params) - res = self.app.get(offset) + res = self.app.get(offset, extra_environ=extra_environ) assert 'Datasets -' in res fv = res.forms['dataset-edit'] prefix = '' fv[prefix + 'name'] = new_name - res = fv.submit('save', status=302) + res = fv.submit('save', status=302, extra_environ=extra_environ) assert not 'Error' in res, res redirected_to = dict(res.headers).get('Location') or dict(res.headers)['location'] expected_redirect_url = expected_redirect.replace('', new_name) @@ -298,7 +298,6 @@ def test_read(self): assert 'russian' in res assert 'david' in res assert 'roger' in res - assert 'State' in res assert 'genre' in res, res assert 'romantic novel' in res, res assert 'original media' in res, res @@ -574,11 +573,12 @@ class TestEdit(TestPackageForm): @classmethod def setup_class(self): CreateTestData.create() + self._reset_data() def setup(self): if not self.res: - self.res = self.app.get(self.offset) + self.res = self.app.get(self.offset,extra_environ=self.extra_environ_admin) model.Session.remove() @classmethod @@ -602,6 +602,9 @@ def _reset_data(self): self.editpkg = model.Package.by_name(self.editpkg_name) self.admin = model.User.by_name(u'testadmin') + + self.extra_environ_admin = {'REMOTE_USER': self.admin.name.encode('utf8')} + self.extra_environ_russianfan = {'REMOTE_USER': 'russianfan'} self.res = None #get's refreshed by setup model.Session.remove() @@ -612,7 +615,7 @@ def teardown_class(self): def test_edit_basic(self): # just the absolute basics try: - self.res = self.app.get(self.offset) + self.res = self.app.get(self.offset,extra_environ=self.extra_environ_admin) assert 'Edit - Datasets' in self.res, self.res new_name = u'new-name' new_title = u'New Title' @@ -620,7 +623,7 @@ def test_edit_basic(self): prefix = '' fv[prefix + 'name'] = new_name fv[prefix + 'title'] = new_title - res = fv.submit('save') + res = fv.submit('save',extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() offset = url_for(controller='package', action='read', id=new_name) @@ -635,7 +638,7 @@ def test_edit_basic(self): def test_edit(self): # just the key fields try: - self.res = self.app.get(self.offset) + self.res = self.app.get(self.offset,extra_environ=self.extra_environ_admin) assert 'Edit - Datasets' in self.res, self.res assert self.editpkg.notes in self.res @@ -653,7 +656,7 @@ def test_edit(self): #fv[prefix + 'resources__0__url'] = new_download_url fv[prefix + 'license_id'] = newlicense_id fv[prefix + 'version'] = newversion - res = fv.submit('save') + res = fv.submit('save',extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() model.Session.remove() @@ -673,7 +676,7 @@ def test_edit_basic_pkg_by_id(self): try: pkg = model.Package.by_name(u'editpkgtest') offset = url_for(controller='package', action='edit', id=pkg.id) - res = self.app.get(offset) + res = self.app.get(offset, extra_environ=self.extra_environ_admin) #assert res.body == self.res.body, self.diff_responses(res, self.res) assert 'Edit - Datasets' in res, res assert pkg.name in res @@ -683,7 +686,7 @@ def test_edit_basic_pkg_by_id(self): prefix = '' fv[prefix + 'name'] = new_name fv[prefix + 'title'] = new_title - res = fv.submit('save') + res = fv.submit('save', extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() offset = url_for(controller='package', action='read', id=new_name) @@ -710,7 +713,7 @@ def test_edit_2_tags_and_groups(self): fv[prefix + 'tag_string'] = tagvalues exp_log_message = 'test_edit_2: making some changes' fv['log_message'] = exp_log_message - res = fv.submit('save') + res = fv.submit('save', extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() assert '%s - Datasets' % self.editpkg_name in res @@ -720,7 +723,7 @@ def test_edit_2_tags_and_groups(self): for tag in newtags: assert tag in outtags rev = model.Revision.youngest(model.Session) - assert rev.author == 'Unknown IP Address' + assert rev.author == self.admin.name assert rev.message == exp_log_message def test_missing_fields(self): @@ -728,12 +731,12 @@ def test_missing_fields(self): # (Spammers can cause this) fv = self.res.forms['dataset-edit'] del fv.fields['log_message'] - res = fv.submit('save', status=400) + res = fv.submit('save', status=400, extra_environ=self.extra_environ_admin) fv = self.res.forms['dataset-edit'] prefix = '' del fv.fields[prefix + 'license_id'] - res = fv.submit('save', status=400) + res = fv.submit('save', status=400, extra_environ=self.extra_environ_admin) def test_redirect_after_edit_using_param(self): return_url = 'http://random.site.com/dataset/?test=param' @@ -741,13 +744,13 @@ def test_redirect_after_edit_using_param(self): # 'http%3A%2F%2Frandom.site.com%2Fdataset%2F%3CNAME%3E%3Ftest%3Dparam' expected_redirect = return_url self._check_redirect(return_url, expected_redirect, - pkg_name_to_edit=self.editpkg_name) + pkg_name_to_edit=self.editpkg_name, extra_environ=self.extra_environ_admin) def test_redirect_after_edit_using_config(self): return_url = '' # redirect comes from test.ini setting expected_redirect = config['package_edit_return_url'] self._check_redirect(return_url, expected_redirect, - pkg_name_to_edit=self.editpkg_name) + pkg_name_to_edit=self.editpkg_name, extra_environ=self.extra_environ_admin) def test_edit_all_fields(self): try: @@ -905,7 +908,7 @@ def test_edit_bad_log_message(self): fv = self.res.forms['dataset-edit'] prefix = '' fv['log_message'] = u'Free enlargements: http://drugs.com/' # spam - res = fv.submit('save') + res = fv.submit('save', extra_environ=self.extra_environ_admin) assert 'Error' in res, res self.check_tag(res, '', + '', '', ] uor_differences = set(uors_str) ^ set(expected_uors_str) From 0f0ed9837999396748955b35c9c3bc1991c8ae6f Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 Jul 2012 18:04:08 +0100 Subject: [PATCH 125/278] Update and clean up libraries --- .../scripts/vendor/moment/1.6.2/moment.js | 918 +++ .../vendor/mustache/0.5.0-dev/mustache.js | 536 ++ .../vendor/recline/css/data-explorer.css | 80 - .../scripts/vendor/recline/css/graph.css | 55 - .../scripts/vendor/recline/css/grid.css | 319 - .../vendor/recline/css/images/edit-map.png | Bin 1569 -> 0 bytes .../recline/css/images/menu-dropdown.png | Bin 1123 -> 0 bytes .../recline/css/images/small-spinner.gif | Bin 1849 -> 0 bytes .../public/scripts/vendor/recline/css/map.css | 28 - .../scripts/vendor/recline/css/recline.css | 633 ++ ckan/public/scripts/vendor/recline/recline.js | 7196 +++++++++-------- .../vendor/slickgrid/2.0.1/MIT-LICENSE.txt | 20 + .../scripts/vendor/slickgrid/2.0.1/README.txt | 16 + .../slickgrid/{ => 2.0.1}/images/sort-asc.gif | Bin .../{ => 2.0.1}/images/sort-desc.gif | Bin .../2.0.1/jquery-ui-1.8.16.custom.min.js | 611 ++ .../2.0.1/jquery.event.drag-2.0.min.js | 6 + .../vendor/slickgrid/2.0.1/slick.grid.css | 158 + .../vendor/slickgrid/2.0.1/slick.grid.min.js | 84 + .../vendor/slickgrid/images/actions.gif | Bin 170 -> 0 bytes .../slickgrid/images/ajax-loader-small.gif | Bin 1849 -> 0 bytes .../vendor/slickgrid/images/arrow_redo.png | Bin 625 -> 0 bytes .../images/arrow_right_peppermint.png | Bin 240 -> 0 bytes .../images/arrow_right_spearmint.png | Bin 240 -> 0 bytes .../vendor/slickgrid/images/arrow_undo.png | Bin 631 -> 0 bytes .../vendor/slickgrid/images/bullet_blue.png | Bin 289 -> 0 bytes .../vendor/slickgrid/images/bullet_star.png | Bin 347 -> 0 bytes .../slickgrid/images/bullet_toggle_minus.png | Bin 207 -> 0 bytes .../slickgrid/images/bullet_toggle_plus.png | Bin 209 -> 0 bytes .../vendor/slickgrid/images/calendar.gif | Bin 1035 -> 0 bytes .../vendor/slickgrid/images/collapse.gif | Bin 846 -> 0 bytes .../slickgrid/images/comment_yellow.gif | Bin 257 -> 0 bytes .../scripts/vendor/slickgrid/images/down.gif | Bin 59 -> 0 bytes .../vendor/slickgrid/images/drag-handle.png | Bin 1223 -> 0 bytes .../slickgrid/images/editor-helper-bg.gif | Bin 1164 -> 0 bytes .../vendor/slickgrid/images/expand.gif | Bin 851 -> 0 bytes .../vendor/slickgrid/images/header-bg.gif | Bin 872 -> 0 bytes .../slickgrid/images/header-columns-bg.gif | Bin 836 -> 0 bytes .../images/header-columns-over-bg.gif | Bin 823 -> 0 bytes .../scripts/vendor/slickgrid/images/help.png | Bin 510 -> 0 bytes .../scripts/vendor/slickgrid/images/info.gif | Bin 80 -> 0 bytes .../vendor/slickgrid/images/listview.gif | Bin 2380 -> 0 bytes .../vendor/slickgrid/images/pencil.gif | Bin 914 -> 0 bytes .../vendor/slickgrid/images/row-over-bg.gif | Bin 823 -> 0 bytes .../vendor/slickgrid/images/sort-asc.png | Bin 163 -> 0 bytes .../vendor/slickgrid/images/sort-desc.png | Bin 161 -> 0 bytes .../vendor/slickgrid/images/stripes.png | Bin 1238 -> 0 bytes .../vendor/slickgrid/images/tag_red.png | Bin 592 -> 0 bytes .../scripts/vendor/slickgrid/images/tick.png | Bin 537 -> 0 bytes .../vendor/slickgrid/images/user_identity.gif | Bin 905 -> 0 bytes .../slickgrid/images/user_identity_plus.gif | Bin 546 -> 0 bytes .../vendor/slickgrid/slick.columnpicker.css | 30 - .../vendor/slickgrid/slick.columnpicker.js | 105 - .../scripts/vendor/slickgrid/slick.grid.css | 153 - .../scripts/vendor/slickgrid/slick.grid.js | 2504 ------ 55 files changed, 6798 insertions(+), 6654 deletions(-) create mode 100644 ckan/public/scripts/vendor/moment/1.6.2/moment.js create mode 100644 ckan/public/scripts/vendor/mustache/0.5.0-dev/mustache.js delete mode 100644 ckan/public/scripts/vendor/recline/css/data-explorer.css delete mode 100644 ckan/public/scripts/vendor/recline/css/graph.css delete mode 100644 ckan/public/scripts/vendor/recline/css/grid.css delete mode 100755 ckan/public/scripts/vendor/recline/css/images/edit-map.png delete mode 100755 ckan/public/scripts/vendor/recline/css/images/menu-dropdown.png delete mode 100755 ckan/public/scripts/vendor/recline/css/images/small-spinner.gif delete mode 100644 ckan/public/scripts/vendor/recline/css/map.css create mode 100644 ckan/public/scripts/vendor/recline/css/recline.css create mode 100644 ckan/public/scripts/vendor/slickgrid/2.0.1/MIT-LICENSE.txt create mode 100644 ckan/public/scripts/vendor/slickgrid/2.0.1/README.txt rename ckan/public/scripts/vendor/slickgrid/{ => 2.0.1}/images/sort-asc.gif (100%) mode change 100755 => 100644 rename ckan/public/scripts/vendor/slickgrid/{ => 2.0.1}/images/sort-desc.gif (100%) mode change 100755 => 100644 create mode 100644 ckan/public/scripts/vendor/slickgrid/2.0.1/jquery-ui-1.8.16.custom.min.js create mode 100644 ckan/public/scripts/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.min.js create mode 100644 ckan/public/scripts/vendor/slickgrid/2.0.1/slick.grid.css create mode 100644 ckan/public/scripts/vendor/slickgrid/2.0.1/slick.grid.min.js delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/actions.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/ajax-loader-small.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/arrow_redo.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/arrow_right_peppermint.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/arrow_right_spearmint.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/arrow_undo.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/bullet_blue.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/bullet_star.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/bullet_toggle_minus.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/bullet_toggle_plus.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/calendar.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/collapse.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/comment_yellow.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/down.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/drag-handle.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/editor-helper-bg.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/expand.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/header-bg.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/header-columns-bg.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/header-columns-over-bg.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/help.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/info.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/listview.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/pencil.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/row-over-bg.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/sort-asc.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/sort-desc.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/stripes.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/tag_red.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/tick.png delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/user_identity.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/images/user_identity_plus.gif delete mode 100755 ckan/public/scripts/vendor/slickgrid/slick.columnpicker.css delete mode 100755 ckan/public/scripts/vendor/slickgrid/slick.columnpicker.js delete mode 100755 ckan/public/scripts/vendor/slickgrid/slick.grid.css delete mode 100755 ckan/public/scripts/vendor/slickgrid/slick.grid.js diff --git a/ckan/public/scripts/vendor/moment/1.6.2/moment.js b/ckan/public/scripts/vendor/moment/1.6.2/moment.js new file mode 100644 index 00000000000..c7901d27706 --- /dev/null +++ b/ckan/public/scripts/vendor/moment/1.6.2/moment.js @@ -0,0 +1,918 @@ +// moment.js +// version : 1.6.2 +// author : Tim Wood +// license : MIT +// momentjs.com + +(function (Date, undefined) { + + var moment, + VERSION = "1.6.2", + round = Math.round, i, + // internal storage for language config files + languages = {}, + currentLanguage = 'en', + + // check for nodeJS + hasModule = (typeof module !== 'undefined'), + + // parameters to check for on the lang config + langConfigProperties = 'months|monthsShort|monthsParse|weekdays|weekdaysShort|longDateFormat|calendar|relativeTime|ordinal|meridiem'.split('|'), + + // ASP.NET json date format regex + aspNetJsonRegex = /^\/?Date\((\-?\d+)/i, + + // format tokens + formattingTokens = /(\[[^\[]*\])|(\\)?(Mo|MM?M?M?|Do|DDDo|DD?D?D?|dddd?|do?|w[o|w]?|YYYY|YY|a|A|hh?|HH?|mm?|ss?|SS?S?|zz?|ZZ?|LT|LL?L?L?)/g, + + // parsing tokens + parseMultipleFormatChunker = /([0-9a-zA-Z\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+)/gi, + + // parsing token regexes + parseTokenOneOrTwoDigits = /\d\d?/, // 0 - 99 + parseTokenOneToThreeDigits = /\d{1,3}/, // 0 - 999 + parseTokenThreeDigits = /\d{3}/, // 000 - 999 + parseTokenFourDigits = /\d{4}/, // 0000 - 9999 + parseTokenWord = /[0-9a-z\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF]+/i, // any word characters or numbers + parseTokenTimezone = /Z|[\+\-]\d\d:?\d\d/i, // +00:00 -00:00 +0000 -0000 or Z + parseTokenT = /T/i, // T (ISO seperator) + + // preliminary iso regex + // 0000-00-00 + T + 00 or 00:00 or 00:00:00 or 00:00:00.000 + +00:00 or +0000 + isoRegex = /^\s*\d{4}-\d\d-\d\d(T(\d\d(:\d\d(:\d\d(\.\d\d?\d?)?)?)?)?([\+\-]\d\d:?\d\d)?)?/, + isoFormat = 'YYYY-MM-DDTHH:mm:ssZ', + + // iso time formats and regexes + isoTimes = [ + ['HH:mm:ss.S', /T\d\d:\d\d:\d\d\.\d{1,3}/], + ['HH:mm:ss', /T\d\d:\d\d:\d\d/], + ['HH:mm', /T\d\d:\d\d/], + ['HH', /T\d\d/] + ], + + // timezone chunker "+10:00" > ["10", "00"] or "-1530" > ["-15", "30"] + parseTimezoneChunker = /([\+\-]|\d\d)/gi, + + // getter and setter names + proxyGettersAndSetters = 'Month|Date|Hours|Minutes|Seconds|Milliseconds'.split('|'), + unitMillisecondFactors = { + 'Milliseconds' : 1, + 'Seconds' : 1e3, + 'Minutes' : 6e4, + 'Hours' : 36e5, + 'Days' : 864e5, + 'Months' : 2592e6, + 'Years' : 31536e6 + }; + + // Moment prototype object + function Moment(date, isUTC) { + this._d = date; + this._isUTC = !!isUTC; + } + + function absRound(number) { + if (number < 0) { + return Math.ceil(number); + } else { + return Math.floor(number); + } + } + + // Duration Constructor + function Duration(duration) { + var data = this._data = {}, + years = duration.years || duration.y || 0, + months = duration.months || duration.M || 0, + weeks = duration.weeks || duration.w || 0, + days = duration.days || duration.d || 0, + hours = duration.hours || duration.h || 0, + minutes = duration.minutes || duration.m || 0, + seconds = duration.seconds || duration.s || 0, + milliseconds = duration.milliseconds || duration.ms || 0; + + // representation for dateAddRemove + this._milliseconds = milliseconds + + seconds * 1e3 + // 1000 + minutes * 6e4 + // 1000 * 60 + hours * 36e5; // 1000 * 60 * 60 + // Because of dateAddRemove treats 24 hours as different from a + // day when working around DST, we need to store them separately + this._days = days + + weeks * 7; + // It is impossible translate months into days without knowing + // which months you are are talking about, so we have to store + // it separately. + this._months = months + + years * 12; + + // The following code bubbles up values, see the tests for + // examples of what that means. + data.milliseconds = milliseconds % 1000; + seconds += absRound(milliseconds / 1000); + + data.seconds = seconds % 60; + minutes += absRound(seconds / 60); + + data.minutes = minutes % 60; + hours += absRound(minutes / 60); + + data.hours = hours % 24; + days += absRound(hours / 24); + + days += weeks * 7; + data.days = days % 30; + + months += absRound(days / 30); + + data.months = months % 12; + years += absRound(months / 12); + + data.years = years; + } + + // left zero fill a number + // see http://jsperf.com/left-zero-filling for performance comparison + function leftZeroFill(number, targetLength) { + var output = number + ''; + while (output.length < targetLength) { + output = '0' + output; + } + return output; + } + + // helper function for _.addTime and _.subtractTime + function addOrSubtractDurationFromMoment(mom, duration, isAdding) { + var ms = duration._milliseconds, + d = duration._days, + M = duration._months, + currentDate; + + if (ms) { + mom._d.setTime(+mom + ms * isAdding); + } + if (d) { + mom.date(mom.date() + d * isAdding); + } + if (M) { + currentDate = mom.date(); + mom.date(1) + .month(mom.month() + M * isAdding) + .date(Math.min(currentDate, mom.daysInMonth())); + } + } + + // check if is an array + function isArray(input) { + return Object.prototype.toString.call(input) === '[object Array]'; + } + + // convert an array to a date. + // the array should mirror the parameters below + // note: all values past the year are optional and will default to the lowest possible value. + // [year, month, day , hour, minute, second, millisecond] + function dateFromArray(input) { + return new Date(input[0], input[1] || 0, input[2] || 1, input[3] || 0, input[4] || 0, input[5] || 0, input[6] || 0); + } + + // format date using native date object + function formatMoment(m, inputString) { + var currentMonth = m.month(), + currentDate = m.date(), + currentYear = m.year(), + currentDay = m.day(), + currentHours = m.hours(), + currentMinutes = m.minutes(), + currentSeconds = m.seconds(), + currentMilliseconds = m.milliseconds(), + currentZone = -m.zone(), + ordinal = moment.ordinal, + meridiem = moment.meridiem; + // check if the character is a format + // return formatted string or non string. + // + // uses switch/case instead of an object of named functions (like http://phpjs.org/functions/date:380) + // for minification and performance + // see http://jsperf.com/object-of-functions-vs-switch for performance comparison + function replaceFunction(input) { + // create a couple variables to be used later inside one of the cases. + var a, b; + switch (input) { + // MONTH + case 'M' : + return currentMonth + 1; + case 'Mo' : + return (currentMonth + 1) + ordinal(currentMonth + 1); + case 'MM' : + return leftZeroFill(currentMonth + 1, 2); + case 'MMM' : + return moment.monthsShort[currentMonth]; + case 'MMMM' : + return moment.months[currentMonth]; + // DAY OF MONTH + case 'D' : + return currentDate; + case 'Do' : + return currentDate + ordinal(currentDate); + case 'DD' : + return leftZeroFill(currentDate, 2); + // DAY OF YEAR + case 'DDD' : + a = new Date(currentYear, currentMonth, currentDate); + b = new Date(currentYear, 0, 1); + return ~~ (((a - b) / 864e5) + 1.5); + case 'DDDo' : + a = replaceFunction('DDD'); + return a + ordinal(a); + case 'DDDD' : + return leftZeroFill(replaceFunction('DDD'), 3); + // WEEKDAY + case 'd' : + return currentDay; + case 'do' : + return currentDay + ordinal(currentDay); + case 'ddd' : + return moment.weekdaysShort[currentDay]; + case 'dddd' : + return moment.weekdays[currentDay]; + // WEEK OF YEAR + case 'w' : + a = new Date(currentYear, currentMonth, currentDate - currentDay + 5); + b = new Date(a.getFullYear(), 0, 4); + return ~~ ((a - b) / 864e5 / 7 + 1.5); + case 'wo' : + a = replaceFunction('w'); + return a + ordinal(a); + case 'ww' : + return leftZeroFill(replaceFunction('w'), 2); + // YEAR + case 'YY' : + return leftZeroFill(currentYear % 100, 2); + case 'YYYY' : + return currentYear; + // AM / PM + case 'a' : + return meridiem ? meridiem(currentHours, currentMinutes, false) : (currentHours > 11 ? 'pm' : 'am'); + case 'A' : + return meridiem ? meridiem(currentHours, currentMinutes, true) : (currentHours > 11 ? 'PM' : 'AM'); + // 24 HOUR + case 'H' : + return currentHours; + case 'HH' : + return leftZeroFill(currentHours, 2); + // 12 HOUR + case 'h' : + return currentHours % 12 || 12; + case 'hh' : + return leftZeroFill(currentHours % 12 || 12, 2); + // MINUTE + case 'm' : + return currentMinutes; + case 'mm' : + return leftZeroFill(currentMinutes, 2); + // SECOND + case 's' : + return currentSeconds; + case 'ss' : + return leftZeroFill(currentSeconds, 2); + // MILLISECONDS + case 'S' : + return ~~ (currentMilliseconds / 100); + case 'SS' : + return leftZeroFill(~~(currentMilliseconds / 10), 2); + case 'SSS' : + return leftZeroFill(currentMilliseconds, 3); + // TIMEZONE + case 'Z' : + return (currentZone < 0 ? '-' : '+') + leftZeroFill(~~(Math.abs(currentZone) / 60), 2) + ':' + leftZeroFill(~~(Math.abs(currentZone) % 60), 2); + case 'ZZ' : + return (currentZone < 0 ? '-' : '+') + leftZeroFill(~~(10 * Math.abs(currentZone) / 6), 4); + // LONG DATES + case 'L' : + case 'LL' : + case 'LLL' : + case 'LLLL' : + case 'LT' : + return formatMoment(m, moment.longDateFormat[input]); + // DEFAULT + default : + return input.replace(/(^\[)|(\\)|\]$/g, ""); + } + } + return inputString.replace(formattingTokens, replaceFunction); + } + + // get the regex to find the next token + function getParseRegexForToken(token) { + switch (token) { + case 'DDDD': + return parseTokenThreeDigits; + case 'YYYY': + return parseTokenFourDigits; + case 'S': + case 'SS': + case 'SSS': + case 'DDD': + return parseTokenOneToThreeDigits; + case 'MMM': + case 'MMMM': + case 'ddd': + case 'dddd': + case 'a': + case 'A': + return parseTokenWord; + case 'Z': + case 'ZZ': + return parseTokenTimezone; + case 'T': + return parseTokenT; + case 'MM': + case 'DD': + case 'dd': + case 'YY': + case 'HH': + case 'hh': + case 'mm': + case 'ss': + case 'M': + case 'D': + case 'd': + case 'H': + case 'h': + case 'm': + case 's': + return parseTokenOneOrTwoDigits; + default : + return new RegExp(token.replace('\\', '')); + } + } + + // function to convert string input to date + function addTimeToArrayFromToken(token, input, datePartArray, config) { + var a; + //console.log('addTime', format, input); + switch (token) { + // MONTH + case 'M' : // fall through to MM + case 'MM' : + datePartArray[1] = (input == null) ? 0 : ~~input - 1; + break; + case 'MMM' : // fall through to MMMM + case 'MMMM' : + for (a = 0; a < 12; a++) { + if (moment.monthsParse[a].test(input)) { + datePartArray[1] = a; + break; + } + } + break; + // DAY OF MONTH + case 'D' : // fall through to DDDD + case 'DD' : // fall through to DDDD + case 'DDD' : // fall through to DDDD + case 'DDDD' : + datePartArray[2] = ~~input; + break; + // YEAR + case 'YY' : + input = ~~input; + datePartArray[0] = input + (input > 70 ? 1900 : 2000); + break; + case 'YYYY' : + datePartArray[0] = ~~Math.abs(input); + break; + // AM / PM + case 'a' : // fall through to A + case 'A' : + config.isPm = ((input + '').toLowerCase() === 'pm'); + break; + // 24 HOUR + case 'H' : // fall through to hh + case 'HH' : // fall through to hh + case 'h' : // fall through to hh + case 'hh' : + datePartArray[3] = ~~input; + break; + // MINUTE + case 'm' : // fall through to mm + case 'mm' : + datePartArray[4] = ~~input; + break; + // SECOND + case 's' : // fall through to ss + case 'ss' : + datePartArray[5] = ~~input; + break; + // MILLISECOND + case 'S' : + case 'SS' : + case 'SSS' : + datePartArray[6] = ~~ (('0.' + input) * 1000); + break; + // TIMEZONE + case 'Z' : // fall through to ZZ + case 'ZZ' : + config.isUTC = true; + a = (input + '').match(parseTimezoneChunker); + if (a && a[1]) { + config.tzh = ~~a[1]; + } + if (a && a[2]) { + config.tzm = ~~a[2]; + } + // reverse offsets + if (a && a[0] === '+') { + config.tzh = -config.tzh; + config.tzm = -config.tzm; + } + break; + } + } + + // date from string and format string + function makeDateFromStringAndFormat(string, format) { + var datePartArray = [0, 0, 1, 0, 0, 0, 0], + config = { + tzh : 0, // timezone hour offset + tzm : 0 // timezone minute offset + }, + tokens = format.match(formattingTokens), + i, parsedInput; + + for (i = 0; i < tokens.length; i++) { + parsedInput = (getParseRegexForToken(tokens[i]).exec(string) || [])[0]; + string = string.replace(getParseRegexForToken(tokens[i]), ''); + addTimeToArrayFromToken(tokens[i], parsedInput, datePartArray, config); + } + // handle am pm + if (config.isPm && datePartArray[3] < 12) { + datePartArray[3] += 12; + } + // if is 12 am, change hours to 0 + if (config.isPm === false && datePartArray[3] === 12) { + datePartArray[3] = 0; + } + // handle timezone + datePartArray[3] += config.tzh; + datePartArray[4] += config.tzm; + // return + return config.isUTC ? new Date(Date.UTC.apply({}, datePartArray)) : dateFromArray(datePartArray); + } + + // compare two arrays, return the number of differences + function compareArrays(array1, array2) { + var len = Math.min(array1.length, array2.length), + lengthDiff = Math.abs(array1.length - array2.length), + diffs = 0, + i; + for (i = 0; i < len; i++) { + if (~~array1[i] !== ~~array2[i]) { + diffs++; + } + } + return diffs + lengthDiff; + } + + // date from string and array of format strings + function makeDateFromStringAndArray(string, formats) { + var output, + inputParts = string.match(parseMultipleFormatChunker) || [], + formattedInputParts, + scoreToBeat = 99, + i, + currentDate, + currentScore; + for (i = 0; i < formats.length; i++) { + currentDate = makeDateFromStringAndFormat(string, formats[i]); + formattedInputParts = formatMoment(new Moment(currentDate), formats[i]).match(parseMultipleFormatChunker) || []; + currentScore = compareArrays(inputParts, formattedInputParts); + if (currentScore < scoreToBeat) { + scoreToBeat = currentScore; + output = currentDate; + } + } + return output; + } + + // date from iso format + function makeDateFromString(string) { + var format = 'YYYY-MM-DDT', + i; + if (isoRegex.exec(string)) { + for (i = 0; i < 4; i++) { + if (isoTimes[i][1].exec(string)) { + format += isoTimes[i][0]; + break; + } + } + return parseTokenTimezone.exec(string) ? + makeDateFromStringAndFormat(string, format + ' Z') : + makeDateFromStringAndFormat(string, format); + } + return new Date(string); + } + + // helper function for moment.fn.from, moment.fn.fromNow, and moment.duration.fn.humanize + function substituteTimeAgo(string, number, withoutSuffix, isFuture) { + var rt = moment.relativeTime[string]; + return (typeof rt === 'function') ? + rt(number || 1, !!withoutSuffix, string, isFuture) : + rt.replace(/%d/i, number || 1); + } + + function relativeTime(milliseconds, withoutSuffix) { + var seconds = round(Math.abs(milliseconds) / 1000), + minutes = round(seconds / 60), + hours = round(minutes / 60), + days = round(hours / 24), + years = round(days / 365), + args = seconds < 45 && ['s', seconds] || + minutes === 1 && ['m'] || + minutes < 45 && ['mm', minutes] || + hours === 1 && ['h'] || + hours < 22 && ['hh', hours] || + days === 1 && ['d'] || + days <= 25 && ['dd', days] || + days <= 45 && ['M'] || + days < 345 && ['MM', round(days / 30)] || + years === 1 && ['y'] || ['yy', years]; + args[2] = withoutSuffix; + args[3] = milliseconds > 0; + return substituteTimeAgo.apply({}, args); + } + + moment = function (input, format) { + if (input === null || input === '') { + return null; + } + var date, + matched, + isUTC; + // parse Moment object + if (moment.isMoment(input)) { + date = new Date(+input._d); + isUTC = input._isUTC; + // parse string and format + } else if (format) { + if (isArray(format)) { + date = makeDateFromStringAndArray(input, format); + } else { + date = makeDateFromStringAndFormat(input, format); + } + // evaluate it as a JSON-encoded date + } else { + matched = aspNetJsonRegex.exec(input); + date = input === undefined ? new Date() : + matched ? new Date(+matched[1]) : + input instanceof Date ? input : + isArray(input) ? dateFromArray(input) : + typeof input === 'string' ? makeDateFromString(input) : + new Date(input); + } + return new Moment(date, isUTC); + }; + + // creating with utc + moment.utc = function (input, format) { + if (isArray(input)) { + return new Moment(new Date(Date.UTC.apply({}, input)), true); + } + return (format && input) ? + moment(input + ' +0000', format + ' Z').utc() : + moment(input && !parseTokenTimezone.exec(input) ? input + '+0000' : input).utc(); + }; + + // creating with unix timestamp (in seconds) + moment.unix = function (input) { + return moment(input * 1000); + }; + + // duration + moment.duration = function (input, key) { + var isDuration = moment.isDuration(input), + isNumber = (typeof input === 'number'), + duration = (isDuration ? input._data : (isNumber ? {} : input)); + + if (isNumber) { + if (key) { + duration[key] = input; + } else { + duration.milliseconds = input; + } + } + + return new Duration(duration); + }; + + // humanizeDuration + // This method is deprecated in favor of the new Duration object. Please + // see the moment.duration method. + moment.humanizeDuration = function (num, type, withSuffix) { + return moment.duration(num, type === true ? null : type).humanize(type === true ? true : withSuffix); + }; + + // version number + moment.version = VERSION; + + // default format + moment.defaultFormat = isoFormat; + + // language switching and caching + moment.lang = function (key, values) { + var i, req, + parse = []; + if (!key) { + return currentLanguage; + } + if (values) { + for (i = 0; i < 12; i++) { + parse[i] = new RegExp('^' + values.months[i] + '|^' + values.monthsShort[i].replace('.', ''), 'i'); + } + values.monthsParse = values.monthsParse || parse; + languages[key] = values; + } + if (languages[key]) { + for (i = 0; i < langConfigProperties.length; i++) { + moment[langConfigProperties[i]] = languages[key][langConfigProperties[i]] || + languages.en[langConfigProperties[i]]; + } + currentLanguage = key; + } else { + if (hasModule) { + req = require('./lang/' + key); + moment.lang(key, req); + } + } + }; + + // set default language + moment.lang('en', { + months : "January_February_March_April_May_June_July_August_September_October_November_December".split("_"), + monthsShort : "Jan_Feb_Mar_Apr_May_Jun_Jul_Aug_Sep_Oct_Nov_Dec".split("_"), + weekdays : "Sunday_Monday_Tuesday_Wednesday_Thursday_Friday_Saturday".split("_"), + weekdaysShort : "Sun_Mon_Tue_Wed_Thu_Fri_Sat".split("_"), + longDateFormat : { + LT : "h:mm A", + L : "MM/DD/YYYY", + LL : "MMMM D YYYY", + LLL : "MMMM D YYYY LT", + LLLL : "dddd, MMMM D YYYY LT" + }, + meridiem : false, + calendar : { + sameDay : '[Today at] LT', + nextDay : '[Tomorrow at] LT', + nextWeek : 'dddd [at] LT', + lastDay : '[Yesterday at] LT', + lastWeek : '[last] dddd [at] LT', + sameElse : 'L' + }, + relativeTime : { + future : "in %s", + past : "%s ago", + s : "a few seconds", + m : "a minute", + mm : "%d minutes", + h : "an hour", + hh : "%d hours", + d : "a day", + dd : "%d days", + M : "a month", + MM : "%d months", + y : "a year", + yy : "%d years" + }, + ordinal : function (number) { + var b = number % 10; + return (~~ (number % 100 / 10) === 1) ? 'th' : + (b === 1) ? 'st' : + (b === 2) ? 'nd' : + (b === 3) ? 'rd' : 'th'; + } + }); + + // compare moment object + moment.isMoment = function (obj) { + return obj instanceof Moment; + }; + + // for typechecking Duration objects + moment.isDuration = function (obj) { + return obj instanceof Duration; + }; + + // shortcut for prototype + moment.fn = Moment.prototype = { + + clone : function () { + return moment(this); + }, + + valueOf : function () { + return +this._d; + }, + + unix : function () { + return Math.floor(+this._d / 1000); + }, + + toString : function () { + return this._d.toString(); + }, + + toDate : function () { + return this._d; + }, + + utc : function () { + this._isUTC = true; + return this; + }, + + local : function () { + this._isUTC = false; + return this; + }, + + format : function (inputString) { + return formatMoment(this, inputString ? inputString : moment.defaultFormat); + }, + + add : function (input, val) { + var dur = val ? moment.duration(+val, input) : moment.duration(input); + addOrSubtractDurationFromMoment(this, dur, 1); + return this; + }, + + subtract : function (input, val) { + var dur = val ? moment.duration(+val, input) : moment.duration(input); + addOrSubtractDurationFromMoment(this, dur, -1); + return this; + }, + + diff : function (input, val, asFloat) { + var inputMoment = this._isUTC ? moment(input).utc() : moment(input).local(), + zoneDiff = (this.zone() - inputMoment.zone()) * 6e4, + diff = this._d - inputMoment._d - zoneDiff, + year = this.year() - inputMoment.year(), + month = this.month() - inputMoment.month(), + date = this.date() - inputMoment.date(), + output; + if (val === 'months') { + output = year * 12 + month + date / 30; + } else if (val === 'years') { + output = year + (month + date / 30) / 12; + } else { + output = val === 'seconds' ? diff / 1e3 : // 1000 + val === 'minutes' ? diff / 6e4 : // 1000 * 60 + val === 'hours' ? diff / 36e5 : // 1000 * 60 * 60 + val === 'days' ? diff / 864e5 : // 1000 * 60 * 60 * 24 + val === 'weeks' ? diff / 6048e5 : // 1000 * 60 * 60 * 24 * 7 + diff; + } + return asFloat ? output : round(output); + }, + + from : function (time, withoutSuffix) { + return moment.duration(this.diff(time)).humanize(!withoutSuffix); + }, + + fromNow : function (withoutSuffix) { + return this.from(moment(), withoutSuffix); + }, + + calendar : function () { + var diff = this.diff(moment().sod(), 'days', true), + calendar = moment.calendar, + allElse = calendar.sameElse, + format = diff < -6 ? allElse : + diff < -1 ? calendar.lastWeek : + diff < 0 ? calendar.lastDay : + diff < 1 ? calendar.sameDay : + diff < 2 ? calendar.nextDay : + diff < 7 ? calendar.nextWeek : allElse; + return this.format(typeof format === 'function' ? format.apply(this) : format); + }, + + isLeapYear : function () { + var year = this.year(); + return (year % 4 === 0 && year % 100 !== 0) || year % 400 === 0; + }, + + isDST : function () { + return (this.zone() < moment([this.year()]).zone() || + this.zone() < moment([this.year(), 5]).zone()); + }, + + day : function (input) { + var day = this._isUTC ? this._d.getUTCDay() : this._d.getDay(); + return input == null ? day : + this.add({ d : input - day }); + }, + + sod: function () { + return moment(this) + .hours(0) + .minutes(0) + .seconds(0) + .milliseconds(0); + }, + + eod: function () { + // end of day = start of day plus 1 day, minus 1 millisecond + return this.sod().add({ + d : 1, + ms : -1 + }); + }, + + zone : function () { + return this._isUTC ? 0 : this._d.getTimezoneOffset(); + }, + + daysInMonth : function () { + return moment(this).month(this.month() + 1).date(0).date(); + } + }; + + // helper for adding shortcuts + function makeGetterAndSetter(name, key) { + moment.fn[name] = function (input) { + var utc = this._isUTC ? 'UTC' : ''; + if (input != null) { + this._d['set' + utc + key](input); + return this; + } else { + return this._d['get' + utc + key](); + } + }; + } + + // loop through and add shortcuts (Month, Date, Hours, Minutes, Seconds, Milliseconds) + for (i = 0; i < proxyGettersAndSetters.length; i ++) { + makeGetterAndSetter(proxyGettersAndSetters[i].toLowerCase(), proxyGettersAndSetters[i]); + } + + // add shortcut for year (uses different syntax than the getter/setter 'year' == 'FullYear') + makeGetterAndSetter('year', 'FullYear'); + + moment.duration.fn = Duration.prototype = { + weeks : function () { + return absRound(this.days() / 7); + }, + + valueOf : function () { + return this._milliseconds + + this._days * 864e5 + + this._months * 2592e6; + }, + + humanize : function (withSuffix) { + var difference = +this, + rel = moment.relativeTime, + output = relativeTime(difference, !withSuffix); + + if (withSuffix) { + output = (difference <= 0 ? rel.past : rel.future).replace(/%s/i, output); + } + + return output; + } + }; + + function makeDurationGetter(name) { + moment.duration.fn[name] = function () { + return this._data[name]; + }; + } + + function makeDurationAsGetter(name, factor) { + moment.duration.fn['as' + name] = function () { + return +this / factor; + }; + } + + for (i in unitMillisecondFactors) { + if (unitMillisecondFactors.hasOwnProperty(i)) { + makeDurationAsGetter(i, unitMillisecondFactors[i]); + makeDurationGetter(i.toLowerCase()); + } + } + + makeDurationAsGetter('Weeks', 6048e5); + + // CommonJS module is defined + if (hasModule) { + module.exports = moment; + } + /*global ender:false */ + if (typeof window !== 'undefined' && typeof ender === 'undefined') { + window.moment = moment; + } + /*global define:false */ + if (typeof define === "function" && define.amd) { + define("moment", [], function () { + return moment; + }); + } +})(Date); diff --git a/ckan/public/scripts/vendor/mustache/0.5.0-dev/mustache.js b/ckan/public/scripts/vendor/mustache/0.5.0-dev/mustache.js new file mode 100644 index 00000000000..641cebd5746 --- /dev/null +++ b/ckan/public/scripts/vendor/mustache/0.5.0-dev/mustache.js @@ -0,0 +1,536 @@ +/*! + * mustache.js - Logic-less {{mustache}} templates with JavaScript + * http://github.com/janl/mustache.js + */ +var Mustache = (typeof module !== "undefined" && module.exports) || {}; + +(function (exports) { + + exports.name = "mustache.js"; + exports.version = "0.5.0-dev"; + exports.tags = ["{{", "}}"]; + exports.parse = parse; + exports.compile = compile; + exports.render = render; + exports.clearCache = clearCache; + + // This is here for backwards compatibility with 0.4.x. + exports.to_html = function (template, view, partials, send) { + var result = render(template, view, partials); + + if (typeof send === "function") { + send(result); + } else { + return result; + } + }; + + var _toString = Object.prototype.toString; + var _isArray = Array.isArray; + var _forEach = Array.prototype.forEach; + var _trim = String.prototype.trim; + + var isArray; + if (_isArray) { + isArray = _isArray; + } else { + isArray = function (obj) { + return _toString.call(obj) === "[object Array]"; + }; + } + + var forEach; + if (_forEach) { + forEach = function (obj, callback, scope) { + return _forEach.call(obj, callback, scope); + }; + } else { + forEach = function (obj, callback, scope) { + for (var i = 0, len = obj.length; i < len; ++i) { + callback.call(scope, obj[i], i, obj); + } + }; + } + + var spaceRe = /^\s*$/; + + function isWhitespace(string) { + return spaceRe.test(string); + } + + var trim; + if (_trim) { + trim = function (string) { + return string == null ? "" : _trim.call(string); + }; + } else { + var trimLeft, trimRight; + + if (isWhitespace("\xA0")) { + trimLeft = /^\s+/; + trimRight = /\s+$/; + } else { + // IE doesn't match non-breaking spaces with \s, thanks jQuery. + trimLeft = /^[\s\xA0]+/; + trimRight = /[\s\xA0]+$/; + } + + trim = function (string) { + return string == null ? "" : + String(string).replace(trimLeft, "").replace(trimRight, ""); + }; + } + + var escapeMap = { + "&": "&", + "<": "<", + ">": ">", + '"': '"', + "'": ''' + }; + + function escapeHTML(string) { + return String(string).replace(/&(?!\w+;)|[<>"']/g, function (s) { + return escapeMap[s] || s; + }); + } + + /** + * Adds the `template`, `line`, and `file` properties to the given error + * object and alters the message to provide more useful debugging information. + */ + function debug(e, template, line, file) { + file = file || "