{% endblock %} - -{% block main_content %} -
-
- {% block form %} -

{{ _('Are you sure you want to delete related item - {name}?').format(name=c.related_dict.title) }}

-

-

- - -
-

- {% endblock %} -
-
-{% endblock %} diff --git a/ckan/templates/related/dashboard.html b/ckan/templates/related/dashboard.html deleted file mode 100644 index 841abe99d8b..00000000000 --- a/ckan/templates/related/dashboard.html +++ /dev/null @@ -1,94 +0,0 @@ -{% extends "page.html" %} - -{% set page = c.page %} -{% set item_count = c.page.item_count %} - -{% block subtitle %}{{ _('Apps & Ideas') }}{% endblock %} - -{% block breadcrumb_content %} -
  • {{ _('Apps & Ideas') }}
  • -{% endblock %} - -{% block primary_content %} -
    -
    -

    - {% block page_heading %}{{ _('Apps & Ideas') }}{% endblock %} -

    - - {% block related_items %} - {% if item_count %} - {% trans first=page.first_item, last=page.last_item, item_count=item_count %} -

    Showing items {{ first }} - {{ last }} of {{ item_count }} related items found

    - {% endtrans %} - {% elif c.filters.type %} - {% trans item_count=item_count %} -

    {{ item_count }} related items found

    - {% endtrans %} - {% else %} -

    {{ _('There have been no apps submitted yet.') }} - {% endif %} - {% endblock %} - - {% block related_list %} - {% if page.items %} - {% snippet "related/snippets/related_list.html", related_items=page.items %} - {% endif %} - {% endblock %} -

    - - {% block page_pagination %} - {{ page.pager() }} - {% endblock %} -
    -{% endblock %} - -{% block secondary_content %} -
    -

    {{ _('What are applications?') }}

    -
    - {% trans %} - These are applications built with the datasets as well as ideas for - things that could be done with them. - {% endtrans %} -
    -
    - -
    -

    {{ _('Filter Results') }}

    -
    - - -
    - - -
    - -
    - - -
    - -
    - -
    - -
    - -
    -
    -
    -{% endblock %} diff --git a/ckan/templates/related/edit.html b/ckan/templates/related/edit.html deleted file mode 100644 index 26c1e49b701..00000000000 --- a/ckan/templates/related/edit.html +++ /dev/null @@ -1,8 +0,0 @@ -{% extends "related/base_form_page.html" %} - -{% block subtitle %}{{ _('Edit related item') }}{% endblock %} - -{# TODO: pass the same context in here so we can create links #} -{% block breadcrumb_item %}{{ h.nav_link(_('Edit Related'), controller='related', action='edit', id=c.id, related_id="") }}{% endblock %} - -{% block page_heading %}{{ _('Edit Related Item') }}{% endblock %} diff --git a/ckan/templates/related/edit_form.html b/ckan/templates/related/edit_form.html deleted file mode 100644 index 92cb16696e8..00000000000 --- a/ckan/templates/related/edit_form.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends "related/snippets/related_form.html" %} - -{% block button_text %} - {% if data.id %} - {{ _('Update') }} - {% else %} - {{ _('Create') }} - {% endif %} -{% endblock %} - -{% block delete_button %} - {% if data.id %} - {{ super() }} - {% endif %} -{% endblock %} diff --git a/ckan/templates/related/new.html b/ckan/templates/related/new.html deleted file mode 100644 index 7fb3ce90633..00000000000 --- a/ckan/templates/related/new.html +++ /dev/null @@ -1,7 +0,0 @@ -{% extends "related/base_form_page.html" %} - -{% block subtitle %}{{ _('Create a related item') }}{% endblock %} - -{% block breadcrumb_item %}{{ h.nav_link(_('Create Related'), controller='related', action='new', id=c.id) }}{% endblock %} - -{% block page_heading %}{{ _('Create Related Item') }}{% endblock %} diff --git a/ckan/templates/related/snippets/related_form.html b/ckan/templates/related/snippets/related_form.html deleted file mode 100644 index 23ab88c1c84..00000000000 --- a/ckan/templates/related/snippets/related_form.html +++ /dev/null @@ -1,35 +0,0 @@ -{% import 'macros/form.html' as form %} - -
    - {% block error_summary %} - {% if error_summary | count %} -
    -

    {{ _('The form contains invalid entries:') }}

    -
      - {% for key, error in error_summary.items() %} -
    • {{ key }}: {{ error }}
    • - {% endfor %} -
    -
    - {% endif %} - {% endblock %} - - {% block fields %} - {{ form.input('title', label=_('Title'), id='field-title', placeholder=_('My Related Item'), value=data.title, error=errors.title, classes=['control-full']) }} - {{ form.input('url', label=_('URL'), id='field-url', placeholder=_('http://example.com/'), value=data.url, error=errors.url, classes=['control-full']) }} - {{ form.input('image_url', label=_('Image URL'), id='field-image-url', placeholder=_('http://example.com/image.png'), value=data.image_url, error=errors.image_url, classes=['control-full']) }} - {{ form.markdown('description', label=_('Description'), id='field-description', placeholder=_('A little information about the item...'), value=data.description, error=errors.description) }} - {{ form.select('type', label=_('Type'), id='field-types', selected=data.type, options=c.types, error=errors.type) }} - {% endblock %} - -
    - {% block delete_button %} - {% if h.check_access('related_delete', {'id': data.id}) %} - {% set locale = h.dump_json({'content': _('Are you sure you want to delete this related item?')}) %} - {% block delete_button_text %}{{ _('Delete') }}{% endblock %} - {% endif %} - {% endblock %} - {{ h.nav_link(_('Cancel'), controller='related', action='list', id=c.id, class_='btn') }} - -
    -
    diff --git a/ckan/templates/related/snippets/related_item.html b/ckan/templates/related/snippets/related_item.html deleted file mode 100644 index 2053f7c0406..00000000000 --- a/ckan/templates/related/snippets/related_item.html +++ /dev/null @@ -1,41 +0,0 @@ -{# -Displays a single related item. - -related - The related item dict. -pkg_id - The id of the owner package. If present the edit button will be - displayed. - -Example: - - - -#} -{% set placeholder_map = { -'application': h.url_for_static('/base/images/placeholder-application.png') -} %} -{% set tooltip = _('Go to {related_item_type}').format(related_item_type=related.type|replace('_', ' ')|title) %} - -{% if position is divisibleby 3 %} -
  • -{% endif %} diff --git a/ckan/templates/related/snippets/related_list.html b/ckan/templates/related/snippets/related_list.html deleted file mode 100644 index 7256ba97dc3..00000000000 --- a/ckan/templates/related/snippets/related_list.html +++ /dev/null @@ -1,18 +0,0 @@ -{# -Renders a list of related item elements - -related_items - A list of related items. -pkg_id - A package id for the items used to determine if the edit button - should be displayed. - -Example: - - - {% snippet "related/snippets/related_list.html", related_items=c.pkg.related, pkg_id=c.pkg.name %} - -#} -
      - {% for related in related_items %} - {% snippet "related/snippets/related_item.html", pkg_id=pkg_id, related=related, position=loop.index %} - {% endfor %} -
    diff --git a/ckan/tests/legacy/functional/test_related.py b/ckan/tests/legacy/functional/test_related.py index bb67323520f..aa636a8957d 100644 --- a/ckan/tests/legacy/functional/test_related.py +++ b/ckan/tests/legacy/functional/test_related.py @@ -10,57 +10,6 @@ import ckan.tests.legacy.functional.api.base as apibase -class TestRelatedUI(base.FunctionalTestCase): - @classmethod - def setup_class(self): - model.Session.remove() - tests.CreateTestData.create() - - @classmethod - def teardown_class(self): - model.repo.rebuild_db() - - def test_related_new(self): - offset = h.url_for(controller='related', - action='new', id='warandpeace') - res = self.app.get(offset, status=200, - extra_environ={"REMOTE_USER": "testsysadmin"}) - assert 'URL' in res, "URL missing in response text" - assert 'Title' in res, "Title missing in response text" - - data = { - "title": "testing_create", - "url": u"http://ckan.org/feed/", - } - res = self.app.post(offset, params=data, - status=[200,302], - extra_environ={"REMOTE_USER": "testsysadmin"}) - - def test_related_new_missing(self): - offset = h.url_for(controller='related', - action='new', id='non-existent dataset') - res = self.app.get(offset, status=404, - extra_environ={"REMOTE_USER": "testsysadmin"}) - - def test_related_new_fail(self): - offset = h.url_for(controller='related', - action='new', id='warandpeace') - print '@@@@', offset - res = self.app.get(offset, status=200, - extra_environ={"REMOTE_USER": "testsysadmin"}) - assert 'URL' in res, "URL missing in response text" - assert 'Title' in res, "Title missing in response text" - - data = { - "title": "testing_create", - } - res = self.app.post(offset, params=data, - status=[200,302], - extra_environ={"REMOTE_USER": "testsysadmin"}) - assert 'error' in res, res - - - class TestRelated: @classmethod @@ -357,7 +306,7 @@ def test_update_related_item_check_owner_status(self): } user = model.User.by_name('tester') admin = model.User.by_name('testsysadmin') - + #create related item context = dict(model=model, user=user.name, session=model.Session) data_dict = dict(title="testing_create",description="description", From 91ddda5392ca84311f0822fdf86a9b81a1b19a23 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Mon, 7 Sep 2015 15:21:46 +0100 Subject: [PATCH 256/307] [#2619] Close form tag in organization form --- ckan/templates/organization/snippets/organization_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/organization/snippets/organization_form.html b/ckan/templates/organization/snippets/organization_form.html index 73fa115e2d2..d78a8e959ee 100644 --- a/ckan/templates/organization/snippets/organization_form.html +++ b/ckan/templates/organization/snippets/organization_form.html @@ -1,6 +1,6 @@ {% import 'macros/form.html' as form %} -
    {% block error_summary %} {{ form.errors(error_summary) }} {% endblock %} From 247dc40315c0f9271e226acd8298de6de4a60d1f Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 8 Sep 2015 08:06:45 +0100 Subject: [PATCH 257/307] Datastore doc improvement * Removed confusing line-break after the pipe * Removed 'postgres' keyword - it was removed in the syntax of recent versions of ckan. --- doc/maintaining/datastore.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index ebd02020e0c..271ad742b99 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -134,13 +134,12 @@ superuser using:: Then you can use this connection to set the permissions:: - sudo ckan datastore set-permissions | - sudo -u postgres psql --set ON_ERROR_STOP=1 + sudo ckan datastore set-permissions | sudo -u postgres psql --set ON_ERROR_STOP=1 .. note:: If you performed a source install, you will need to replace all references to ``sudo ckan ...`` with ``paster --plugin=ckan ...`` and provide the path to - the config file, e.g. ``paster --plugin=ckan datastore set-permissions postgres -c /etc/ckan/default/development.ini`` + the config file, e.g. ``paster --plugin=ckan datastore set-permissions -c /etc/ckan/default/development.ini`` If your database server is not local, but you can access it over SSH, you can pipe the permissions script over SSH:: From 156348c1d11a5ceb556f593d4094074f912842a6 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 8 Sep 2015 12:52:58 +0100 Subject: [PATCH 258/307] Stop reraising exceptions caused by notifications. They are secondary errors and should not prevent the change to the dataset occurring. e.g. I am harvesting some data, but because datapusher has an exception, every time the harvester tries to write it has an exception and fails. Admins should look at their logs for these exceptions, and the primary functions should carry on working. --- ckan/model/modification.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/ckan/model/modification.py b/ckan/model/modification.py index 585d771198b..6a5213170a2 100644 --- a/ckan/model/modification.py +++ b/ckan/model/modification.py @@ -9,6 +9,7 @@ __all__ = ['DomainObjectModificationExtension'] + class DomainObjectModificationExtension(plugins.SingletonPlugin): """ A domain object level interface to change notifications @@ -30,7 +31,6 @@ def notify_observers(self, func): plugins.IDomainObjectModification): func(observer) - def before_commit(self, session): self.notify_observers(session, self.notify) @@ -60,7 +60,8 @@ def notify_observers(self, session, method): for item in plugins.PluginImplementations(plugins.IResourceUrlChange): item.notify(obj) - changed_pkgs = set(obj for obj in changed if isinstance(obj, _package.Package)) + changed_pkgs = set(obj for obj in changed + if isinstance(obj, _package.Package)) for obj in new | changed | deleted: if not isinstance(obj, _package.Package): @@ -76,7 +77,6 @@ def notify_observers(self, session, method): for obj in changed_pkgs: method(obj, domain_object.DomainObjectOperation.changed) - def notify(self, entity, operation): for observer in plugins.PluginImplementations( plugins.IDomainObjectModification): @@ -84,9 +84,6 @@ def notify(self, entity, operation): observer.notify(entity, operation) except Exception, ex: log.exception(ex) - # We reraise all exceptions so they are obvious there - # is something wrong - raise def notify_after_commit(self, entity, operation): for observer in plugins.PluginImplementations( @@ -95,6 +92,3 @@ def notify_after_commit(self, entity, operation): observer.notify_after_commit(entity, operation) except Exception, ex: log.exception(ex) - # We reraise all exceptions so they are obvious there - # is something wrong - raise From f19705d92da36d77988c7f6ebed6983d3bd0e043 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 8 Sep 2015 14:33:42 +0100 Subject: [PATCH 259/307] Make sure that SearchIndexErrors *are* raised CKAN relies on the SearchIndexError being raised in the notify() call of IDomainObjectNotification. It still only logs most Exceptions, but explicitly re-raises SearchIndexErrors as they are fatal to CKAN. --- ckan/model/modification.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ckan/model/modification.py b/ckan/model/modification.py index 6a5213170a2..d8bbcaad665 100644 --- a/ckan/model/modification.py +++ b/ckan/model/modification.py @@ -1,5 +1,7 @@ import logging +from ckan.lib.search import SearchIndexError + import ckan.plugins as plugins import domain_object import package as _package @@ -82,6 +84,9 @@ def notify(self, entity, operation): plugins.IDomainObjectModification): try: observer.notify(entity, operation) + except SearchIndexError, search_error: + log.exception(search_error) + raise search_error except Exception, ex: log.exception(ex) @@ -90,5 +95,8 @@ def notify_after_commit(self, entity, operation): plugins.IDomainObjectModification): try: observer.notify_after_commit(entity, operation) + except SearchIndexError, search_error: + log.exception(search_error) + raise search_error except Exception, ex: log.exception(ex) From 0f516efcf8356c19493ec261b654449c27d2ce88 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 8 Sep 2015 14:56:32 +0000 Subject: [PATCH 260/307] Added comment to explain. --- ckan/model/modification.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ckan/model/modification.py b/ckan/model/modification.py index d8bbcaad665..c06c72902b9 100644 --- a/ckan/model/modification.py +++ b/ckan/model/modification.py @@ -86,9 +86,13 @@ def notify(self, entity, operation): observer.notify(entity, operation) except SearchIndexError, search_error: log.exception(search_error) + # Reraise, since it's pretty crucial to ckan if it can't index + # a dataset raise search_error except Exception, ex: log.exception(ex) + # Don't reraise other exceptions since they are generally of + # secondary importance so shouldn't disrupt the commit. def notify_after_commit(self, entity, operation): for observer in plugins.PluginImplementations( @@ -97,6 +101,10 @@ def notify_after_commit(self, entity, operation): observer.notify_after_commit(entity, operation) except SearchIndexError, search_error: log.exception(search_error) + # Reraise, since it's pretty crucial to ckan if it can't index + # a dataset raise search_error except Exception, ex: log.exception(ex) + # Don't reraise other exceptions since they are generally of + # secondary importance so shouldn't disrupt the commit. From f065c5e0b0e3a9c403c42374a479d2a069a60081 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Wed, 9 Sep 2015 10:45:59 +0100 Subject: [PATCH 261/307] [#2543] Better test name and comment --- ckan/tests/controllers/test_user.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/tests/controllers/test_user.py b/ckan/tests/controllers/test_user.py index cda6ac6168e..945e8bc6f5a 100644 --- a/ckan/tests/controllers/test_user.py +++ b/ckan/tests/controllers/test_user.py @@ -125,8 +125,12 @@ def test_registered_user_login_bad_password(self): class TestLogout(helpers.FunctionalTestBase): - def test_user_logout(self): - '''_logout url redirects to logged out page.''' + def test_user_logout_url_redirect(self): + '''_logout url redirects to logged out page. + + Note: this doesn't test the actual logout of a logged in user, just + the associated redirect. + ''' app = self._get_test_app() logout_url = url_for(controller='user', action='logout') From 969f7c92fd354d1c3cff765c0f97c93561a46644 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Wed, 9 Sep 2015 15:02:20 +0100 Subject: [PATCH 262/307] pep8 --- ckan/controllers/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index 319f395d81a..84f7e0af437 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -509,11 +509,11 @@ def search(self, ver=None, register=None): else: return self._finish_bad_request( _("Missing search term ('since_id=UUID' or " + - " 'since_time=TIMESTAMP')")) + " 'since_time=TIMESTAMP')")) revs = model.Session.query(model.Revision) \ .filter(model.Revision.timestamp > since_time) \ .order_by(model.Revision.timestamp) \ - .limit(50) # reasonable enough for a page + .limit(50) # reasonable enough for a page return self._finish_ok([rev.id for rev in revs]) elif register in ['dataset', 'package', 'resource']: try: From 58c6e8f05a7c8e683234655138205b1f4439b017 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 10 Sep 2015 09:46:59 +0100 Subject: [PATCH 263/307] Add note about the file uploads code --- ckan/controllers/storage.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py index 10c44cffda9..030f9e9ccee 100644 --- a/ckan/controllers/storage.py +++ b/ckan/controllers/storage.py @@ -1,3 +1,10 @@ +''' + +Note: This is the old file store controller for CKAN < 2.2. +If you are looking for how the file uploads work, you should check `lib/uploader.py` +and the `resource_download` method of the package controller. + +''' import os import re import urllib From 929f0a2d4499d9af7b9b80bc9e2af55a859482ef Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 10 Sep 2015 10:27:04 +0100 Subject: [PATCH 264/307] Fix PEP8 --- ckan/controllers/storage.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/controllers/storage.py b/ckan/controllers/storage.py index 030f9e9ccee..fab34671725 100644 --- a/ckan/controllers/storage.py +++ b/ckan/controllers/storage.py @@ -1,13 +1,13 @@ ''' Note: This is the old file store controller for CKAN < 2.2. -If you are looking for how the file uploads work, you should check `lib/uploader.py` -and the `resource_download` method of the package controller. +If you are looking for how the file uploads work, you should check +`lib/uploader.py` and the `resource_download` method of the package +controller. ''' import os import re -import urllib from ofs import get_impl from paste.fileapp import FileApp From a9c6c0e62a97675dba8f84a5ee58ae3f7b733824 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 15:35:37 +0000 Subject: [PATCH 265/307] [#1572] dataset_purge action added --- ckan/lib/cli.py | 8 ++-- ckan/logic/action/delete.py | 42 ++++++++++++++++++ ckan/logic/auth/delete.py | 4 ++ ckan/model/group.py | 8 ++-- ckan/tests/logic/action/test_delete.py | 60 ++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 9 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index c7831b4e313..f1355633321 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -890,7 +890,6 @@ class DatasetCmd(CkanCommand): def command(self): self._load_config() - import ckan.model as model if not self.args: print self.usage @@ -939,13 +938,12 @@ def delete(self, dataset_ref): print '%s %s -> %s' % (dataset.name, old_state, dataset.state) def purge(self, dataset_ref): - import ckan.model as model dataset = self._get_dataset(dataset_ref) name = dataset.name - rev = model.repo.new_revision() - dataset.purge() - model.repo.commit_and_remove() + context = {'user': self.site_user['name']} + logic.get_action('dataset_purge')( + context, {'id': dataset_ref}) print '%s purged' % name diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index dd562201f19..44b26d64887 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -44,6 +44,9 @@ def user_delete(context, data_dict): def package_delete(context, data_dict): '''Delete a dataset (package). + This makes the dataset disappear from all web & API views, apart from the + trash. + You must be authorized to delete the dataset. :param id: the id or name of the dataset to delete @@ -73,6 +76,45 @@ def package_delete(context, data_dict): entity.delete() model.repo.commit() + +def dataset_purge(context, data_dict): + '''Purge a dataset. + + .. warning:: Purging a dataset cannot be undone! + + Purging a database completely removes the dataset from the CKAN database, + whereias deleting a dataset simply marks the dataset as deleted (it will no + longer show up in the front-end, but is still in the db). + + You must be authorized to purge the dataset. + + :param id: the name or id of the dataset to be purged + :type id: string + + ''' + model = context['model'] + id = _get_or_bust(data_dict, 'id') + + pkg = model.Package.get(id) + context['package'] = pkg + if pkg is None: + raise NotFound('Dataset was not found') + + _check_access('dataset_purge', context, data_dict) + + members = model.Session.query(model.Member) \ + .filter(model.Member.table_id == pkg.id) \ + .filter(model.Member.table_name == 'package') + if members.count() > 0: + for m in members.all(): + m.purge() + + pkg = model.Package.get(id) + model.repo.new_revision() + pkg.purge() + model.repo.commit_and_remove() + + def resource_delete(context, data_dict): '''Delete a resource from a dataset. diff --git a/ckan/logic/auth/delete.py b/ckan/logic/auth/delete.py index 1c94918c394..7978ac4dbbd 100644 --- a/ckan/logic/auth/delete.py +++ b/ckan/logic/auth/delete.py @@ -17,6 +17,10 @@ def package_delete(context, data_dict): # are essentially changing the state field return _auth_update.package_update(context, data_dict) +def dataset_purge(context, data_dict): + # Only sysadmins are authorized to purge datasets + return {'success': False} + def resource_delete(context, data_dict): model = context['model'] user = context.get('user') diff --git a/ckan/model/group.py b/ckan/model/group.py index 0882e9e245b..e9380b1c8ad 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -104,11 +104,11 @@ def related_packages(self): def __unicode__(self): # refer to objects by name, not ID, to help debugging if self.table_name == 'package': - table_info = 'package=%s' % meta.Session.query(_package.Package).\ - get(self.table_id).name + pkg = meta.Session.query(_package.Package).get(self.table_id) + table_info = 'package=%s' % pkg.name if pkg else 'None' elif self.table_name == 'group': - table_info = 'group=%s' % meta.Session.query(Group).\ - get(self.table_id).name + group = meta.Session.query(Group).get(self.table_id) + table_info = 'group=%s' % group.name if group else 'None' else: table_info = 'table_name=%s table_id=%s' % (self.table_name, self.table_id) diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index d8079968bee..826295cc9ad 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -140,3 +140,63 @@ def test_tag_delete_with_unicode_returns_unicode_error(self): assert u'Delta symbol: \u0394' in unicode(e) else: assert 0, 'Should have raised NotFound' + + +class TestDatasetPurge(object): + def setup(self): + helpers.reset_db() + + def test_a_non_sysadmin_cant_purge_dataset(self): + user = factories.User() + dataset = factories.Dataset(user=user) + + assert_raises(logic.NotAuthorized, + helpers.call_action, + 'dataset_purge', + context={'user': user['name'], 'ignore_auth': False}, + id=dataset['name']) + + def test_purged_dataset_does_not_show(self): + dataset = factories.Dataset() + + helpers.call_action('dataset_purge', + context={'ignore_auth': True}, + id=dataset['name']) + + assert_raises(logic.NotFound, helpers.call_action, 'package_show', + context={}, id=dataset['name']) + + def test_purged_dataset_leaves_no_trace_in_the_model(self): + factories.Group(name='group1') + dataset = factories.Dataset( + tags=[{'name': 'tag1'}], + groups=[{'name': 'group1'}], + extras=[{'key': 'testkey', 'value': 'testvalue'}]) + num_revisions_before = model.Session.query(model.Revision).count() + + helpers.call_action('dataset_purge', + context={'ignore_auth': True}, + id=dataset['name']) + num_revisions_after = model.Session.query(model.Revision).count() + + # the Package and related objects are gone + assert_equals(model.Session.query(model.Package).all(), []) + assert_equals(model.Session.query(model.PackageTag).all(), []) + # there is no clean-up of the tag object itself, just the PackageTag. + assert_equals([t.name for t in model.Session.query(model.Tag).all()], + ['tag1']) + assert_equals(model.Session.query(model.PackageExtra).all(), []) + # the only member left is the user created by factories.Group() + assert_equals([m.table_name + for m in model.Session.query(model.Member).all()], + ['user']) + + # all the object revisions were purged too + assert_equals(model.Session.query(model.PackageRevision).all(), []) + assert_equals(model.Session.query(model.PackageTagRevision).all(), []) + assert_equals(model.Session.query(model.PackageExtraRevision).all(), + []) + # Member is not revisioned + + # No Revision objects were purged, in fact 1 is created for the purge + assert_equals(num_revisions_after - num_revisions_before, 1) From e2fedb6a6870b4c7329899f10c897addd216d9b5 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 17:28:42 +0100 Subject: [PATCH 266/307] [#2631] Also purge Members by table_id. * purge members instead of delete - I see no reason to keep the Member as state=deleted. * a blank revision was being created - comment explains the removal. --- ckan/logic/action/delete.py | 12 ++++-- ckan/tests/logic/action/test_delete.py | 52 ++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index dd562201f19..791b34a3d3e 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -1,5 +1,7 @@ '''API functions for deleting data from CKAN.''' +import sqlalchemy as sqla + import ckan.logic import ckan.logic.action import ckan.plugins as plugins @@ -397,12 +399,14 @@ def _group_or_org_purge(context, data_dict, is_org=False): else: _check_access('group_purge', context, data_dict) - members = model.Session.query(model.Member) - members = members.filter(model.Member.group_id == group.id) + members = model.Session.query(model.Member) \ + .filter(sqla.or_(model.Member.group_id == group.id, + model.Member.table_id == group.id)) if members.count() > 0: - model.repo.new_revision() + # no need to do new_revision() because Member is not revisioned, nor + # does it cascade delete any revisioned objects for m in members.all(): - m.delete() + m.purge() model.repo.commit_and_remove() group = model.Group.get(id) diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index d8079968bee..0fec0ee58cc 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -140,3 +140,55 @@ def test_tag_delete_with_unicode_returns_unicode_error(self): assert u'Delta symbol: \u0394' in unicode(e) else: assert 0, 'Should have raised NotFound' + + +class TestGroupPurge(object): + def setup(self): + helpers.reset_db() + + def test_purged_group_does_not_show(self): + group = factories.Group() + + helpers.call_action('group_purge', + context={'ignore_auth': True}, + id=group['name']) + + assert_raises(logic.NotFound, helpers.call_action, 'group_show', + context={}, id=group['name']) + + def test_purged_group_leaves_no_trace_in_the_model(self): + factories.Group(name='parent') + user = factories.User() + group1 = factories.Group(name='group1', + extras=[{'key': 'key1', 'value': 'val1'}], + users=[{'name': user['name']}], + groups=[{'name': 'parent'}]) + factories.Group(name='child', groups=[{'name': 'group1'}]) + num_revisions_before = model.Session.query(model.Revision).count() + + helpers.call_action('group_purge', + context={'ignore_auth': True}, + id=group1['name']) + num_revisions_after = model.Session.query(model.Revision).count() + + # the Group and related objects are gone + assert_equals(sorted([g.name for g in + model.Session.query(model.Group).all()]), + ['child', 'parent']) + assert_equals(model.Session.query(model.GroupExtra).all(), []) + # the only members left are the users for the parent and child + assert_equals(sorted([ + (m.table_name, m.group.name) + for m in model.Session.query(model.Member).join(model.Group)]), + [('user', 'child'), ('user', 'parent')]) + + # the group's object revisions were purged too + assert_equals(sorted( + [gr.name for gr in model.Session.query(model.GroupRevision)]), + ['child', 'parent']) + assert_equals(model.Session.query(model.GroupExtraRevision).all(), + []) + # Member is not revisioned + + # No Revision objects were purged, in fact 1 is created for the purge + assert_equals(num_revisions_after - num_revisions_before, 1) From 3bbef1b029dcec224c54bafddc00655ac234916d Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 17:39:36 +0100 Subject: [PATCH 267/307] [#2631] Tests added for auth and orgs. --- ckan/tests/logic/action/test_delete.py | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 0fec0ee58cc..1303fd3b1cc 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -146,6 +146,16 @@ class TestGroupPurge(object): def setup(self): helpers.reset_db() + def test_a_non_sysadmin_cant_purge_group(self): + user = factories.User() + group = factories.Group(user=user) + + assert_raises(logic.NotAuthorized, + helpers.call_action, + 'group_purge', + context={'user': user['name'], 'ignore_auth': False}, + id=group['name']) + def test_purged_group_does_not_show(self): group = factories.Group() @@ -192,3 +202,66 @@ def test_purged_group_leaves_no_trace_in_the_model(self): # No Revision objects were purged, in fact 1 is created for the purge assert_equals(num_revisions_after - num_revisions_before, 1) + + +class TestOrganizationPurge(object): + def setup(self): + helpers.reset_db() + + def test_a_non_sysadmin_cant_purge_org(self): + user = factories.User() + org = factories.Organization(user=user) + + assert_raises(logic.NotAuthorized, + helpers.call_action, + 'organization_purge', + context={'user': user['name'], 'ignore_auth': False}, + id=org['name']) + + def test_purged_org_does_not_show(self): + org = factories.Organization() + + helpers.call_action('organization_purge', + context={'ignore_auth': True}, + id=org['name']) + + assert_raises(logic.NotFound, helpers.call_action, 'organization_show', + context={}, id=org['name']) + + def test_purged_organization_leaves_no_trace_in_the_model(self): + factories.Organization(name='parent') + user = factories.User() + org1 = factories.Organization( + name='org1', + extras=[{'key': 'key1', 'value': 'val1'}], + users=[{'name': user['name']}], + groups=[{'name': 'parent'}]) + factories.Organization(name='child', groups=[{'name': 'group1'}]) + num_revisions_before = model.Session.query(model.Revision).count() + + helpers.call_action('organization_purge', + context={'ignore_auth': True}, + id=org1['name']) + num_revisions_after = model.Session.query(model.Revision).count() + + # the Organization and related objects are gone + assert_equals(sorted([o.name for o in + model.Session.query(model.Group).all()]), + ['child', 'parent']) + assert_equals(model.Session.query(model.GroupExtra).all(), []) + # the only members left are the users for the parent and child + assert_equals(sorted([ + (m.table_name, m.group.name) + for m in model.Session.query(model.Member).join(model.Group)]), + [('user', 'child'), ('user', 'parent')]) + + # the organization's object revisions were purged too + assert_equals(sorted( + [gr.name for gr in model.Session.query(model.GroupRevision)]), + ['child', 'parent']) + assert_equals(model.Session.query(model.GroupExtraRevision).all(), + []) + # Member is not revisioned + + # No Revision objects were purged, in fact 1 is created for the purge + assert_equals(num_revisions_after - num_revisions_before, 1) From 2f6b9039533ad2c218e22153c19b04edf693cea3 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 21:22:36 +0100 Subject: [PATCH 268/307] [#2632] Delete the owner_org field of orphaned datasets. More tests. --- ckan/logic/action/delete.py | 26 ++++++++ ckan/tests/logic/action/test_delete.py | 89 ++++++++++++++++++++++---- 2 files changed, 102 insertions(+), 13 deletions(-) diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 791b34a3d3e..bf4b4f14416 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -6,6 +6,7 @@ import ckan.logic.action import ckan.plugins as plugins import ckan.lib.dictization.model_dictize as model_dictize +from ckan import authz from ckan.common import _ @@ -399,6 +400,26 @@ def _group_or_org_purge(context, data_dict, is_org=False): else: _check_access('group_purge', context, data_dict) + if is_org: + # Clear the owner_org field + datasets = model.Session.query(model.Package) \ + .filter_by(owner_org=group.id) \ + .filter(model.Package.state != 'deleted') \ + .count() + if datasets: + if not authz.check_config_permission('ckan.auth.create_unowned_dataset'): + raise ValidationError('Organization cannot be purged while it ' + 'still has datasets') + pkg_table = model.package_table + # using Core SQLA instead of the ORM should be faster + model.Session.execute( + pkg_table.update().where( + sqla.and_(pkg_table.c.owner_org == group.id, + pkg_table.c.state != 'deleted') + ).values(owner_org=None) + ) + + # Delete related Memberships members = model.Session.query(model.Member) \ .filter(sqla.or_(model.Member.group_id == group.id, model.Member.table_id == group.id)) @@ -423,6 +444,8 @@ def group_purge(context, data_dict): whereas deleting a group simply marks the group as deleted (it will no longer show up in the frontend, but is still in the db). + Datasets in the organization will remain, just not in the purged group. + You must be authorized to purge the group. :param id: the name or id of the group to be purged @@ -441,6 +464,9 @@ def organization_purge(context, data_dict): deleted (it will no longer show up in the frontend, but is still in the db). + Datasets owned by the organization will remain, just not in an + organization any more. + You must be authorized to purge the organization. :param id: the name or id of the organization to be purged diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 1303fd3b1cc..b7640854160 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -5,6 +5,7 @@ import ckan.logic as logic import ckan.model as model import ckan.plugins as p +import ckan.lib.search as search assert_equals = nose.tools.assert_equals assert_raises = nose.tools.assert_raises @@ -159,13 +160,42 @@ def test_a_non_sysadmin_cant_purge_group(self): def test_purged_group_does_not_show(self): group = factories.Group() - helpers.call_action('group_purge', - context={'ignore_auth': True}, - id=group['name']) + helpers.call_action('group_purge', id=group['name']) assert_raises(logic.NotFound, helpers.call_action, 'group_show', context={}, id=group['name']) + def test_purged_group_is_not_listed(self): + group = factories.Group() + + helpers.call_action('group_purge', id=group['name']) + + assert_equals(helpers.call_action('group_list', context={}), []) + + def test_dataset_in_a_purged_group_no_longer_shows_that_group(self): + group = factories.Group() + dataset = factories.Dataset(groups=[{'name': group['name']}]) + + helpers.call_action('group_purge', id=group['name']) + + dataset_shown = helpers.call_action('package_show', context={}, + id=dataset['id']) + assert_equals(dataset_shown['groups'], []) + + def test_purged_group_is_not_in_search_results_for_its_ex_dataset(self): + search.clear() + group = factories.Group() + dataset = factories.Dataset(groups=[{'name': group['name']}]) + def get_search_result_groups(): + results = helpers.call_action('package_search', + q=dataset['title'])['results'] + return [g['name'] for g in results[0]['groups']] + assert_equals(get_search_result_groups(), [group['name']]) + + helpers.call_action('group_purge', id=group['name']) + + assert_equals(get_search_result_groups(), []) + def test_purged_group_leaves_no_trace_in_the_model(self): factories.Group(name='parent') user = factories.User() @@ -173,12 +203,11 @@ def test_purged_group_leaves_no_trace_in_the_model(self): extras=[{'key': 'key1', 'value': 'val1'}], users=[{'name': user['name']}], groups=[{'name': 'parent'}]) + factories.Dataset(name='ds', groups=[{'name': 'group1'}]) factories.Group(name='child', groups=[{'name': 'group1'}]) num_revisions_before = model.Session.query(model.Revision).count() - helpers.call_action('group_purge', - context={'ignore_auth': True}, - id=group1['name']) + helpers.call_action('group_purge', id=group1['name']) num_revisions_after = model.Session.query(model.Revision).count() # the Group and related objects are gone @@ -191,6 +220,9 @@ def test_purged_group_leaves_no_trace_in_the_model(self): (m.table_name, m.group.name) for m in model.Session.query(model.Member).join(model.Group)]), [('user', 'child'), ('user', 'parent')]) + # the dataset is still there though + assert_equals([p.name for p in model.Session.query(model.Package)], + ['ds']) # the group's object revisions were purged too assert_equals(sorted( @@ -221,13 +253,42 @@ def test_a_non_sysadmin_cant_purge_org(self): def test_purged_org_does_not_show(self): org = factories.Organization() - helpers.call_action('organization_purge', - context={'ignore_auth': True}, - id=org['name']) + helpers.call_action('organization_purge', id=org['name']) assert_raises(logic.NotFound, helpers.call_action, 'organization_show', context={}, id=org['name']) + def test_purged_org_is_not_listed(self): + org = factories.Organization() + + helpers.call_action('organization_purge', id=org['name']) + + assert_equals(helpers.call_action('organization_list', context={}), []) + + def test_dataset_in_a_purged_org_no_longer_shows_that_org(self): + org = factories.Organization() + dataset = factories.Dataset(owner_org=org['id']) + + helpers.call_action('organization_purge', id=org['name']) + + dataset_shown = helpers.call_action('package_show', context={}, + id=dataset['id']) + assert_equals(dataset_shown['owner_org'], None) + + def test_purged_org_is_not_in_search_results_for_its_ex_dataset(self): + search.clear() + org = factories.Organization() + dataset = factories.Dataset(owner_org=org['id']) + def get_search_result_owner_org(): + results = helpers.call_action('package_search', + q=dataset['title'])['results'] + return results[0]['owner_org'] + assert_equals(get_search_result_owner_org(), org['id']) + + helpers.call_action('organization_purge', id=org['name']) + + assert_equals(get_search_result_owner_org(), None) + def test_purged_organization_leaves_no_trace_in_the_model(self): factories.Organization(name='parent') user = factories.User() @@ -236,12 +297,11 @@ def test_purged_organization_leaves_no_trace_in_the_model(self): extras=[{'key': 'key1', 'value': 'val1'}], users=[{'name': user['name']}], groups=[{'name': 'parent'}]) - factories.Organization(name='child', groups=[{'name': 'group1'}]) + factories.Dataset(name='ds', owner_org=org1['id']) + factories.Organization(name='child', groups=[{'name': 'org1'}]) num_revisions_before = model.Session.query(model.Revision).count() - helpers.call_action('organization_purge', - context={'ignore_auth': True}, - id=org1['name']) + helpers.call_action('organization_purge', id=org1['name']) num_revisions_after = model.Session.query(model.Revision).count() # the Organization and related objects are gone @@ -254,6 +314,9 @@ def test_purged_organization_leaves_no_trace_in_the_model(self): (m.table_name, m.group.name) for m in model.Session.query(model.Member).join(model.Group)]), [('user', 'child'), ('user', 'parent')]) + # the dataset is still there though + assert_equals([p.name for p in model.Session.query(model.Package)], + ['ds']) # the organization's object revisions were purged too assert_equals(sorted( From d5d275d399be99664c653b2cc5d1ca198823326f Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 21:30:22 +0100 Subject: [PATCH 269/307] [#2632/#2631] All tests migrated now from legacy. --- .../test_group_and_organization_purge.py | 409 ------------------ ckan/tests/logic/action/test_delete.py | 16 + 2 files changed, 16 insertions(+), 409 deletions(-) delete mode 100644 ckan/tests/legacy/functional/api/model/test_group_and_organization_purge.py diff --git a/ckan/tests/legacy/functional/api/model/test_group_and_organization_purge.py b/ckan/tests/legacy/functional/api/model/test_group_and_organization_purge.py deleted file mode 100644 index b7f6b626851..00000000000 --- a/ckan/tests/legacy/functional/api/model/test_group_and_organization_purge.py +++ /dev/null @@ -1,409 +0,0 @@ -'''Functional tests for the group_ and organization_purge APIs. - -''' -import ckan.model as model -import ckan.tests.legacy as tests - -import paste -import pylons.test - - -class TestGroupAndOrganizationPurging(object): - '''Tests for the group_ and organization_purge APIs. - - ''' - @classmethod - def setup_class(cls): - cls.app = paste.fixture.TestApp(pylons.test.pylonsapp) - - # Make a sysadmin user. - cls.sysadmin = model.User(name='test_sysadmin', sysadmin=True) - model.Session.add(cls.sysadmin) - model.Session.commit() - model.Session.remove() - - # A package that will be added to our test groups and organizations. - cls.package = tests.call_action_api(cls.app, 'package_create', - name='test_package', - apikey=cls.sysadmin.apikey) - - # A user who will not be a member of our test groups or organizations. - cls.visitor = tests.call_action_api(cls.app, 'user_create', - name='non_member', - email='blah', - password='farm', - apikey=cls.sysadmin.apikey) - - # A user who will become a member of our test groups and organizations. - cls.member = tests.call_action_api(cls.app, 'user_create', - name='member', - email='blah', - password='farm', - apikey=cls.sysadmin.apikey) - - # A user who will become an editor of our test groups and - # organizations. - cls.editor = tests.call_action_api(cls.app, 'user_create', - name='editor', - email='blah', - password='farm', - apikey=cls.sysadmin.apikey) - - # A user who will become an admin of our test groups and organizations. - cls.admin = tests.call_action_api(cls.app, 'user_create', - name='admin', - email='blah', - password='farm', - apikey=cls.sysadmin.apikey) - - @classmethod - def teardown_class(cls): - model.repo.rebuild_db() - - def _organization_create(self, organization_name): - '''Return an organization with some users and a dataset.''' - - # Make an organization with some users. - users = [{'name': self.member['name'], 'capacity': 'member'}, - {'name': self.editor['name'], 'capacity': 'editor'}, - {'name': self.admin['name'], 'capacity': 'admin'}] - organization = tests.call_action_api(self.app, 'organization_create', - apikey=self.sysadmin.apikey, - name=organization_name, - users=users) - - # Add a dataset to the organization (have to do this separately - # because the packages param of organization_create doesn't work). - tests.call_action_api(self.app, 'package_update', - name=self.package['name'], - owner_org=organization['name'], - apikey=self.sysadmin.apikey) - - return organization - - def _group_create(self, group_name): - '''Return a group with some users and a dataset.''' - - # Make a group with some users and a dataset. - group = tests.call_action_api(self.app, 'group_create', - apikey=self.sysadmin.apikey, - name=group_name, - users=[ - {'name': self.member['name'], - 'capacity': 'member', - }, - {'name': self.editor['name'], - 'capacity': 'editor', - }, - {'name': self.admin['name'], - 'capacity': 'admin', - }], - packages=[ - {'id': self.package['name']}], - ) - - return group - - def _test_group_or_organization_purge(self, name, by_id, is_org): - '''Create a group or organization with the given name, and test - purging it. - - :param name: the name of the group or organization to create and purge - :param by_id: if True, pass the organization's id to - organization_purge, otherwise pass its name - :type by_id: boolean - :param is_org: if True create and purge an organization, if False a - group - :type is_org: boolean - - ''' - if is_org: - group_or_org = self._organization_create(name) - else: - group_or_org = self._group_create(name) - - # Purge the group or organization. - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - if by_id: - identifier = group_or_org['id'] - else: - identifier = group_or_org['name'] - result = tests.call_action_api(self.app, action, id=identifier, - apikey=self.sysadmin.apikey, - ) - assert result is None - - # Now trying to show the group or organization should give a 404. - if is_org: - action = 'organization_show' - else: - action = 'group_show' - result = tests.call_action_api(self.app, action, id=name, status=404) - assert result == {'__type': 'Not Found Error', 'message': 'Not found'} - - # The group or organization should not appear in group_list or - # organization_list. - if is_org: - action = 'organization_list' - else: - action = 'group_list' - assert name not in tests.call_action_api(self.app, action) - - # The package should no longer belong to the group or organization. - package = tests.call_action_api(self.app, 'package_show', - id=self.package['name']) - if is_org: - assert package['organization'] is None - else: - assert group_or_org['name'] not in [group_['name'] for group_ - in package['groups']] - - # TODO: Also want to assert that user is not in group or organization - # anymore, but how to get a user's groups or organizations? - - # It should be possible to create a new group or organization with the - # same name as the purged one (you would not be able to do this if you - # had merely deleted the original group or organization). - if is_org: - action = 'organization_create' - else: - action = 'group_create' - new_group_or_org = tests.call_action_api(self.app, action, name=name, - apikey=self.sysadmin.apikey, - ) - assert new_group_or_org['name'] == name - - # TODO: Should we do a model-level check, to check that the group or - # org is really purged? - - def test_organization_purge_by_name(self): - '''A sysadmin should be able to purge an organization by name.''' - - self._test_group_or_organization_purge('organization-to-be-purged', - by_id=False, is_org=True) - - def test_group_purge_by_name(self): - '''A sysadmin should be able to purge a group by name.''' - self._test_group_or_organization_purge('group-to-be-purged', - by_id=False, is_org=False) - - def test_organization_purge_by_id(self): - '''A sysadmin should be able to purge an organization by id.''' - self._test_group_or_organization_purge('organization-to-be-purged-2', - by_id=True, is_org=True) - - def test_group_purge_by_id(self): - '''A sysadmin should be able to purge a group by id.''' - self._test_group_or_organization_purge('group-to-be-purged-2', - by_id=True, is_org=False) - - def _test_group_or_org_purge_with_invalid_id(self, is_org): - - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - - for name in ('foo', 'invalid name', None, ''): - # Try to purge an organization, but pass an invalid name. - result = tests.call_action_api(self.app, action, - apikey=self.sysadmin.apikey, - id=name, - status=404, - ) - if is_org: - message = 'Not found: Organization was not found' - else: - message = 'Not found: Group was not found' - assert result == {'__type': 'Not Found Error', 'message': message} - - def test_organization_purge_with_invalid_id(self): - ''' - Trying to purge an organization with an invalid ID should give a 404. - - ''' - self._test_group_or_org_purge_with_invalid_id(is_org=True) - - def test_group_purge_with_invalid_id(self): - '''Trying to purge a group with an invalid ID should give a 404.''' - self._test_group_or_org_purge_with_invalid_id(is_org=False) - - def _test_group_or_org_purge_with_missing_id(self, is_org): - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - result = tests.call_action_api(self.app, action, - apikey=self.sysadmin.apikey, - status=409, - ) - assert result == {'__type': 'Validation Error', - 'id': ['Missing value']} - - def test_organization_purge_with_missing_id(self): - '''Trying to purge an organization without passing an id should give - a 409.''' - self._test_group_or_org_purge_with_missing_id(is_org=True) - - def test_group_purge_with_missing_id(self): - '''Trying to purge a group without passing an id should give a 409.''' - self._test_group_or_org_purge_with_missing_id(is_org=False) - - def _test_visitors_cannot_purge_groups_or_orgs(self, is_org): - if is_org: - group_or_org = self._organization_create('org-to-be-purged-3') - else: - group_or_org = self._group_create('group-to-be-purged-3') - - # Try to purge the group or organization without an API key. - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - result = tests.call_action_api(self.app, action, id=group_or_org['id'], - status=403, - ) - assert result['__type'] == 'Authorization Error' - - def test_visitors_cannot_purge_organizations(self): - '''Visitors (who aren't logged in) should not be authorized to purge - organizations. - - ''' - self._test_visitors_cannot_purge_groups_or_orgs(is_org=True) - - def test_visitors_cannot_purge_groups(self): - '''Visitors (who aren't logged in) should not be authorized to purge - groups. - - ''' - self._test_visitors_cannot_purge_groups_or_orgs(is_org=False) - - def _test_users_cannot_purge_groups_or_orgs(self, is_org): - if is_org: - group_or_org = self._organization_create('org-to-be-purged-4') - else: - group_or_org = self._group_create('group-to-be-purged-4') - - # Try to purge the group or organization with a non-member's API key. - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - result = tests.call_action_api(self.app, action, id=group_or_org['id'], - apikey=self.visitor['apikey'], - status=403, - ) - assert result == {'__type': 'Authorization Error', - 'message': 'Access denied'} - - def test_users_cannot_purge_organizations(self): - '''Users who are not members of an organization should not be - authorized to purge the organization. - - ''' - self._test_users_cannot_purge_groups_or_orgs(is_org=True) - - def test_users_cannot_purge_groups(self): - '''Users who are not members of a group should not be authorized to - purge the group. - - ''' - self._test_users_cannot_purge_groups_or_orgs(is_org=False) - - def _test_members_cannot_purge_groups_or_orgs(self, is_org): - if is_org: - group_or_org = self._organization_create('org-to-be-purged-5') - else: - group_or_org = self._group_create('group-to-be-purged-5') - - # Try to purge the organization with an organization member's API key. - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - result = tests.call_action_api(self.app, action, id=group_or_org['id'], - apikey=self.member['apikey'], - status=403, - ) - assert result == {'__type': 'Authorization Error', - 'message': 'Access denied'} - - def test_members_cannot_purge_organizations(self): - '''Members of an organization should not be authorized to purge the - organization. - - ''' - self._test_members_cannot_purge_groups_or_orgs(is_org=True) - - def test_members_cannot_purge_groups(self): - '''Members of a group should not be authorized to purge the group. - - ''' - self._test_members_cannot_purge_groups_or_orgs(is_org=False) - - def _test_editors_cannot_purge_groups_or_orgs(self, is_org): - if is_org: - group_or_org = self._organization_create('org-to-be-purged-6') - else: - group_or_org = self._group_create('group-to-be-purged-6') - - # Try to purge the group or organization with an editor's API key. - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - result = tests.call_action_api(self.app, action, id=group_or_org['id'], - apikey=self.editor['apikey'], - status=403, - ) - assert result == {'__type': 'Authorization Error', - 'message': 'Access denied'} - - def test_editors_cannot_purge_organizations(self): - '''Editors of an organization should not be authorized to purge the - organization. - - ''' - self._test_editors_cannot_purge_groups_or_orgs(is_org=True) - - def test_editors_cannot_purge_groups(self): - '''Editors of a group should not be authorized to purge the group. - - ''' - self._test_editors_cannot_purge_groups_or_orgs(is_org=False) - - def _test_admins_cannot_purge_groups_or_orgs(self, is_org): - if is_org: - group_or_org = self._organization_create('org-to-be-purged-7') - else: - group_or_org = self._group_create('group-to-be-purged-7') - - # Try to purge the group or organization with an admin's API key. - if is_org: - action = 'organization_purge' - else: - action = 'group_purge' - result = tests.call_action_api(self.app, action, - id=group_or_org['id'], - apikey=self.admin['apikey'], - status=403, - ) - assert result == {'__type': 'Authorization Error', - 'message': 'Access denied'} - - def test_admins_cannot_purge_organizations(self): - '''Admins of an organization should not be authorized to purge the - organization. - - ''' - self._test_admins_cannot_purge_groups_or_orgs(is_org=True) - - def test_admins_cannot_purge_groups(self): - '''Admins of a group should not be authorized to purge the group. - - ''' - self._test_admins_cannot_purge_groups_or_orgs(is_org=False) diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index b7640854160..00d263032bb 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -235,6 +235,14 @@ def test_purged_group_leaves_no_trace_in_the_model(self): # No Revision objects were purged, in fact 1 is created for the purge assert_equals(num_revisions_after - num_revisions_before, 1) + def test_missing_id_returns_error(self): + assert_raises(logic.ValidationError, + helpers.call_action, 'group_purge') + + def test_bad_id_returns_404(self): + assert_raises(logic.NotFound, + helpers.call_action, 'group_purge', id='123') + class TestOrganizationPurge(object): def setup(self): @@ -328,3 +336,11 @@ def test_purged_organization_leaves_no_trace_in_the_model(self): # No Revision objects were purged, in fact 1 is created for the purge assert_equals(num_revisions_after - num_revisions_before, 1) + + def test_missing_id_returns_error(self): + assert_raises(logic.ValidationError, + helpers.call_action, 'organization_purge') + + def test_bad_id_returns_404(self): + assert_raises(logic.NotFound, + helpers.call_action, 'organization_purge', id='123') From 31a28dd62fdd765ac1931463338076758982db9a Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 21:54:09 +0100 Subject: [PATCH 270/307] [#1572] More tests. --- ckan/tests/logic/action/test_delete.py | 51 ++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 826295cc9ad..60e0d00c6dd 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -5,6 +5,7 @@ import ckan.logic as logic import ckan.model as model import ckan.plugins as p +import ckan.lib.search as search assert_equals = nose.tools.assert_equals assert_raises = nose.tools.assert_raises @@ -166,11 +167,44 @@ def test_purged_dataset_does_not_show(self): assert_raises(logic.NotFound, helpers.call_action, 'package_show', context={}, id=dataset['name']) + def test_purged_dataset_is_not_listed(self): + dataset = factories.Dataset() + + helpers.call_action('dataset_purge', id=dataset['name']) + + assert_equals(helpers.call_action('package_list', context={}), []) + + def test_group_no_longer_shows_its_purged_dataset(self): + group = factories.Group() + dataset = factories.Dataset(groups=[{'name': group['name']}]) + + helpers.call_action('dataset_purge', id=dataset['name']) + + dataset_shown = helpers.call_action('group_show', context={}, + id=group['id'], + include_datasets=True) + assert_equals(dataset_shown['packages'], []) + + def test_purged_dataset_is_not_in_search_results(self): + search.clear() + dataset = factories.Dataset() + def get_search_results(): + results = helpers.call_action('package_search', + q=dataset['title'])['results'] + return [d['name'] for d in results] + assert_equals(get_search_results(), [dataset['name']]) + + helpers.call_action('dataset_purge', id=dataset['name']) + + assert_equals(get_search_results(), []) + def test_purged_dataset_leaves_no_trace_in_the_model(self): factories.Group(name='group1') + org = factories.Organization() dataset = factories.Dataset( tags=[{'name': 'tag1'}], groups=[{'name': 'group1'}], + owner_org=org['id'], extras=[{'key': 'testkey', 'value': 'testvalue'}]) num_revisions_before = model.Session.query(model.Revision).count() @@ -186,10 +220,11 @@ def test_purged_dataset_leaves_no_trace_in_the_model(self): assert_equals([t.name for t in model.Session.query(model.Tag).all()], ['tag1']) assert_equals(model.Session.query(model.PackageExtra).all(), []) - # the only member left is the user created by factories.Group() - assert_equals([m.table_name - for m in model.Session.query(model.Member).all()], - ['user']) + # the only member left is for the user created in factories.Group() and + # factories.Organization() + assert_equals([(m.table_name, m.group.name) + for m in model.Session.query(model.Member).join(model.Group)], + [('user', 'group1'), ('user', 'test_org_0')]) # all the object revisions were purged too assert_equals(model.Session.query(model.PackageRevision).all(), []) @@ -200,3 +235,11 @@ def test_purged_dataset_leaves_no_trace_in_the_model(self): # No Revision objects were purged, in fact 1 is created for the purge assert_equals(num_revisions_after - num_revisions_before, 1) + + def test_missing_id_returns_error(self): + assert_raises(logic.ValidationError, + helpers.call_action, 'dataset_purge') + + def test_bad_id_returns_404(self): + assert_raises(logic.NotFound, + helpers.call_action, 'dataset_purge', id='123') From 945aad58088bfea8bdeb18ab2b68738ad96e008a Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 22:44:07 +0100 Subject: [PATCH 271/307] [#1572] Fix slow purging using index. Fix CLI. --- ckan/lib/cli.py | 4 +++- ckan/logic/action/delete.py | 2 +- ckan/migration/versions/079_resource_revision_index.py | 8 ++++++++ 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 ckan/migration/versions/079_resource_revision_index.py diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index f1355633321..1747eade27e 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -938,10 +938,12 @@ def delete(self, dataset_ref): print '%s %s -> %s' % (dataset.name, old_state, dataset.state) def purge(self, dataset_ref): + import ckan.logic as logic dataset = self._get_dataset(dataset_ref) name = dataset.name - context = {'user': self.site_user['name']} + site_user = logic.get_action('get_site_user')({'ignore_auth': True}, {}) + context = {'user': site_user['name']} logic.get_action('dataset_purge')( context, {'id': dataset_ref}) print '%s purged' % name diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 44b26d64887..1d741f6875e 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -83,7 +83,7 @@ def dataset_purge(context, data_dict): .. warning:: Purging a dataset cannot be undone! Purging a database completely removes the dataset from the CKAN database, - whereias deleting a dataset simply marks the dataset as deleted (it will no + whereas deleting a dataset simply marks the dataset as deleted (it will no longer show up in the front-end, but is still in the db). You must be authorized to purge the dataset. diff --git a/ckan/migration/versions/079_resource_revision_index.py b/ckan/migration/versions/079_resource_revision_index.py new file mode 100644 index 00000000000..92744533e4e --- /dev/null +++ b/ckan/migration/versions/079_resource_revision_index.py @@ -0,0 +1,8 @@ +def upgrade(migrate_engine): + migrate_engine.execute( + ''' + CREATE INDEX idx_resource_continuity_id + ON resource_revision (continuity_id); + ''' + ) + From f48a8e67b53933ae3eeab63193b04969587793fc Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 22:51:18 +0100 Subject: [PATCH 272/307] [#1572] Test fixes. --- ckan/migration/versions/079_resource_revision_index.py | 1 - ckan/tests/logic/action/test_delete.py | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ckan/migration/versions/079_resource_revision_index.py b/ckan/migration/versions/079_resource_revision_index.py index 92744533e4e..70d985bfce0 100644 --- a/ckan/migration/versions/079_resource_revision_index.py +++ b/ckan/migration/versions/079_resource_revision_index.py @@ -5,4 +5,3 @@ def upgrade(migrate_engine): ON resource_revision (continuity_id); ''' ) - diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 60e0d00c6dd..37875c94a90 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -188,6 +188,7 @@ def test_group_no_longer_shows_its_purged_dataset(self): def test_purged_dataset_is_not_in_search_results(self): search.clear() dataset = factories.Dataset() + def get_search_results(): results = helpers.call_action('package_search', q=dataset['title'])['results'] @@ -222,9 +223,10 @@ def test_purged_dataset_leaves_no_trace_in_the_model(self): assert_equals(model.Session.query(model.PackageExtra).all(), []) # the only member left is for the user created in factories.Group() and # factories.Organization() - assert_equals([(m.table_name, m.group.name) - for m in model.Session.query(model.Member).join(model.Group)], - [('user', 'group1'), ('user', 'test_org_0')]) + assert_equals(sorted( + [(m.table_name, m.group.name) + for m in model.Session.query(model.Member).join(model.Group)]), + [('user', 'group1'), ('user', org['name'])]) # all the object revisions were purged too assert_equals(model.Session.query(model.PackageRevision).all(), []) From 7ca003ff0bbd156d2b8ba44f872a48d1de39b3a4 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 10 Sep 2015 23:14:51 +0100 Subject: [PATCH 273/307] [#2632] PEP8 --- ckan/tests/logic/action/test_delete.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 00d263032bb..6d3935a34ed 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -186,6 +186,7 @@ def test_purged_group_is_not_in_search_results_for_its_ex_dataset(self): search.clear() group = factories.Group() dataset = factories.Dataset(groups=[{'name': group['name']}]) + def get_search_result_groups(): results = helpers.call_action('package_search', q=dataset['title'])['results'] @@ -287,6 +288,7 @@ def test_purged_org_is_not_in_search_results_for_its_ex_dataset(self): search.clear() org = factories.Organization() dataset = factories.Dataset(owner_org=org['id']) + def get_search_result_owner_org(): results = helpers.call_action('package_search', q=dataset['title'])['results'] From 1d885896e99ec73747574f338f99da207bedf2f1 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 11 Sep 2015 14:29:48 +0100 Subject: [PATCH 274/307] [#2624] Better err handling resource_status_show. check auth should be before attempting the celery import. Also wrapped the sqlalchemy statement in a try/except to catch exceptions raised when celery is installed, but the celery_taskmeta table hasn't been created. --- ckan/logic/action/get.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 377a83dbac9..3244a2920f8 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1198,6 +1198,7 @@ def resource_view_list(context, data_dict): ] return model_dictize.resource_view_list_dictize(resource_views, context) + def resource_status_show(context, data_dict): '''Return the statuses of a resource's tasks. @@ -1207,6 +1208,9 @@ def resource_status_show(context, data_dict): :rtype: list of (status, date_done, traceback, task_status) dictionaries ''' + + _check_access('resource_status_show', context, data_dict) + try: import ckan.lib.celery_app as celery_app except ImportError: @@ -1215,8 +1219,6 @@ def resource_status_show(context, data_dict): model = context['model'] id = _get_or_bust(data_dict, 'id') - _check_access('resource_status_show', context, data_dict) - # needs to be text query as celery tables are not in our model q = _text(""" select status, date_done, traceback, task_status.* @@ -1225,7 +1227,12 @@ def resource_status_show(context, data_dict): and key = 'celery_task_id' where entity_id = :entity_id """) - result = model.Session.connection().execute(q, entity_id=id) + try: + result = model.Session.connection().execute(q, entity_id=id) + except sqlalchemy.exc.ProgrammingError: + # celery tables (celery_taskmeta) may not be created even with celery + # installed, causing ProgrammingError exception. + return {'message': 'queue tables not installed on this instance'} result_list = [_table_dictize(row, context) for row in result] return result_list From 323c5bcd26dbb1da1a8ff66f096bde764bb1cb49 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 7 Aug 2015 15:07:22 +0100 Subject: [PATCH 275/307] Allows the disabling of datastore_search_sql Although the datastore is useful, it may be that some users may will to disable the specific datastore_search_sql. This commit allows them to do that. --- ckanext/datastore/plugin.py | 9 ++++++++- ckanext/datastore/tests/test_disable.py | 27 +++++++++++++++++++++++++ doc/maintaining/configuration.rst | 14 +++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 ckanext/datastore/tests/test_disable.py diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index a7db61dbd52..b581fb1aeac 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -81,6 +81,10 @@ def configure(self, config): # datastore runs on PG prior to 9.0 (for example 8.4). self.legacy_mode = _is_legacy_mode(self.config) + # Check whether users have disabled datastore_search_sql + self.enable_sql_search = p.toolkit.asbool( + self.config.get('ckan.datastore.sqlsearch.enabled', True)) + datapusher_formats = config.get('datapusher.formats', '').split() self.datapusher_formats = datapusher_formats or DEFAULT_FORMATS @@ -246,8 +250,11 @@ def get_actions(self): 'datastore_info': action.datastore_info, } if not self.legacy_mode: + if self.enable_sql_search: + # Only enable search_sql if the config does not disable it + actions.update({'datastore_search_sql': + action.datastore_search_sql}) actions.update({ - 'datastore_search_sql': action.datastore_search_sql, 'datastore_make_private': action.datastore_make_private, 'datastore_make_public': action.datastore_make_public}) return actions diff --git a/ckanext/datastore/tests/test_disable.py b/ckanext/datastore/tests/test_disable.py new file mode 100644 index 00000000000..bc3a6f223b5 --- /dev/null +++ b/ckanext/datastore/tests/test_disable.py @@ -0,0 +1,27 @@ + +import pylons.config as config +import ckan.plugins as p +import nose.tools as t + + +class TestDisable(object): + + @classmethod + def setup_class(cls): + with p.use_plugin('datastore') as the_plugin: + legacy = the_plugin.legacy_mode + + if legacy: + raise nose.SkipTest("SQL tests are not supported in legacy mode") + + @t.raises(KeyError) + def test_disable_sql_search(self): + config['ckan.datastore.sqlsearch.enabled'] = False + with p.use_plugin('datastore') as the_plugin: + print p.toolkit.get_action('datastore_search_sql') + config['ckan.datastore.sqlsearch.enabled'] = True + + def test_enabled_sql_search(self): + config['ckan.datastore.sqlsearch.enabled'] = True + with p.use_plugin('datastore') as the_plugin: + p.toolkit.get_action('datastore_search_sql') diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index 57830ff41c8..62c14ed9125 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -256,6 +256,20 @@ The default method used when creating full-text search indexes. Currently it can be "gin" or "gist". Refer to PostgreSQL's documentation to understand the characteristics of each one and pick the best for your instance. +.. _ckan.datastore.sqlsearch.enabled: + +ckan.datastore.sqlsearch.enabled +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.datastore.sqlsearch.enabled = False + +Default value: ``True`` + +This option allows you to disable the datastore_search_sql action function, and +corresponding API endpoint if you do not wish it to be activated. + Site Settings ------------- From 7096d6e979acb20b0a0ec025c6d8c7ac5f329246 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 11 Sep 2015 17:25:32 +0100 Subject: [PATCH 276/307] [#1572] Improvements * no revision is needed when purging the package. The only thing being created is a Activity, which is not revisioned. * no need to explicitly delete an object's revision objects. Despite the very old comment, it is clear that this does happen naturally through cascades. This is verified in the test and by reading the SQL produced. * Added a resource to the test for fullness. * Added helpful comments to the test-core.ini to show people how to use it to see generated SQL commands. --- ckan/logic/action/delete.py | 3 ++- .../migration/versions/080_continuity_id_indexes.py | 13 +++++++++++++ ckan/model/domain_object.py | 7 ++----- ckan/tests/logic/action/test_delete.py | 7 +++++-- test-core.ini | 5 ++++- 5 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 ckan/migration/versions/080_continuity_id_indexes.py diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 1d741f6875e..e8b70d40968 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -110,7 +110,8 @@ def dataset_purge(context, data_dict): m.purge() pkg = model.Package.get(id) - model.repo.new_revision() + # no new_revision() needed since there are no object_revisions created + # during purge pkg.purge() model.repo.commit_and_remove() diff --git a/ckan/migration/versions/080_continuity_id_indexes.py b/ckan/migration/versions/080_continuity_id_indexes.py new file mode 100644 index 00000000000..1b223ecf47d --- /dev/null +++ b/ckan/migration/versions/080_continuity_id_indexes.py @@ -0,0 +1,13 @@ +def upgrade(migrate_engine): + migrate_engine.execute( + ''' + CREATE INDEX idx_member_continuity_id + ON member_revision (continuity_id); + CREATE INDEX idx_package_tag_continuity_id + ON package_tag_revision (continuity_id); + CREATE INDEX idx_package_continuity_id + ON package_revision (continuity_id); + CREATE INDEX idx_package_extra_continuity_id + ON package_extra_revision (continuity_id); + ''' + ) diff --git a/ckan/model/domain_object.py b/ckan/model/domain_object.py index 4237d77aea7..9e3a2a287f9 100644 --- a/ckan/model/domain_object.py +++ b/ckan/model/domain_object.py @@ -76,15 +76,12 @@ def remove(self): self.Session.remove() def delete(self): + # stateful objects have this method overridden - see + # vmd.base.StatefulObjectMixin self.Session.delete(self) def purge(self): self.Session().autoflush = False - if hasattr(self, '__revisioned__'): # only for versioned objects ... - # this actually should auto occur due to cascade relationships but - # ... - for rev in self.all_revisions: - self.Session.delete(rev) self.Session.delete(self) def as_dict(self): diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 37875c94a90..8aeacca68cf 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -207,6 +207,7 @@ def test_purged_dataset_leaves_no_trace_in_the_model(self): groups=[{'name': 'group1'}], owner_org=org['id'], extras=[{'key': 'testkey', 'value': 'testvalue'}]) + factories.Resource(package_id=dataset['id']) num_revisions_before = model.Session.query(model.Revision).count() helpers.call_action('dataset_purge', @@ -216,6 +217,7 @@ def test_purged_dataset_leaves_no_trace_in_the_model(self): # the Package and related objects are gone assert_equals(model.Session.query(model.Package).all(), []) + assert_equals(model.Session.query(model.Resource).all(), []) assert_equals(model.Session.query(model.PackageTag).all(), []) # there is no clean-up of the tag object itself, just the PackageTag. assert_equals([t.name for t in model.Session.query(model.Tag).all()], @@ -230,13 +232,14 @@ def test_purged_dataset_leaves_no_trace_in_the_model(self): # all the object revisions were purged too assert_equals(model.Session.query(model.PackageRevision).all(), []) + assert_equals(model.Session.query(model.ResourceRevision).all(), []) assert_equals(model.Session.query(model.PackageTagRevision).all(), []) assert_equals(model.Session.query(model.PackageExtraRevision).all(), []) # Member is not revisioned - # No Revision objects were purged, in fact 1 is created for the purge - assert_equals(num_revisions_after - num_revisions_before, 1) + # No Revision objects were purged or created + assert_equals(num_revisions_after - num_revisions_before, 0) def test_missing_id_returns_error(self): assert_raises(logic.ValidationError, diff --git a/test-core.ini b/test-core.ini index 4a0aa72580f..0602a2e40e4 100644 --- a/test-core.ini +++ b/test-core.ini @@ -127,7 +127,10 @@ level = INFO [logger_sqlalchemy] handlers = qualname = sqlalchemy.engine -level = WARN +level = WARNING +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARNING" logs neither. [handler_console] class = StreamHandler From d2c1b8429c56b5c4c68891330803cbab6c1bbad5 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Fri, 11 Sep 2015 18:52:17 +0100 Subject: [PATCH 277/307] Fix encoding of setup.py for non-ascii names Sets the default-encoding so that what the user enters is utf8 and this stops the paster templates for exploding. Also sets the encoding on the setup.py template so that it can be run if it has utf8 chars in it This should fix #2636 --- ckan/pastertemplates/__init__.py | 5 ++++- ckan/pastertemplates/template/setup.py_tmpl | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ckan/pastertemplates/__init__.py b/ckan/pastertemplates/__init__.py index ce5eb5e5741..eea0ebe6923 100644 --- a/ckan/pastertemplates/__init__.py +++ b/ckan/pastertemplates/__init__.py @@ -51,6 +51,9 @@ class CkanextTemplate(Template): def check_vars(self, vars, cmd): vars = Template.check_vars(self, vars, cmd) + reload(sys) + sys.setdefaultencoding('utf-8') + if not vars['project'].startswith('ckanext-'): print "\nError: Project name must start with 'ckanext-'" sys.exit(1) @@ -63,7 +66,7 @@ def check_vars(self, vars, cmd): keywords = [keyword for keyword in keywords if keyword not in ('ckan', 'CKAN')] keywords.insert(0, 'CKAN') - vars['keywords'] = ' '.join(keywords) + vars['keywords'] = u' '.join(keywords) # For an extension named ckanext-example we want a plugin class # named ExamplePlugin. diff --git a/ckan/pastertemplates/template/setup.py_tmpl b/ckan/pastertemplates/template/setup.py_tmpl index e1bc849280b..b3dd34a21a0 100644 --- a/ckan/pastertemplates/template/setup.py_tmpl +++ b/ckan/pastertemplates/template/setup.py_tmpl @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from setuptools import setup, find_packages # Always prefer setuptools over distutils from codecs import open # To use a consistent encoding from os import path From 2c5a25e6b5036af799aae5af9825fe20d17cce52 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Mon, 14 Sep 2015 15:58:07 +0100 Subject: [PATCH 278/307] [#2640] Add offset param to organization_activity --- ckan/config/routing.py | 2 +- ckan/templates/organization/read_base.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 9cfab2d6393..2350b9cccd5 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -316,7 +316,7 @@ def make_map(): 'member_delete', 'history' ]))) - m.connect('organization_activity', '/organization/activity/{id}', + m.connect('organization_activity', '/organization/activity/{id}/{offset}', action='activity', ckan_icon='time') m.connect('organization_read', '/organization/{id}', action='read') m.connect('organization_about', '/organization/about/{id}', diff --git a/ckan/templates/organization/read_base.html b/ckan/templates/organization/read_base.html index 1c9454f4729..866debf7b81 100644 --- a/ckan/templates/organization/read_base.html +++ b/ckan/templates/organization/read_base.html @@ -15,7 +15,7 @@ {% block content_primary_nav %} {{ h.build_nav_icon('organization_read', _('Datasets'), id=c.group_dict.name) }} - {{ h.build_nav_icon('organization_activity', _('Activity Stream'), id=c.group_dict.name) }} + {{ h.build_nav_icon('organization_activity', _('Activity Stream'), id=c.group_dict.name, offset=0) }} {{ h.build_nav_icon('organization_about', _('About'), id=c.group_dict.name) }} {% endblock %} From 03c1ce053359603d69d9de4e8a81238171354082 Mon Sep 17 00:00:00 2001 From: Denis Zgonjanin Date: Mon, 14 Sep 2015 14:26:20 -0400 Subject: [PATCH 279/307] Search index rebuild takes a while; let's give it a progress counter --- ckan/lib/search/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index ea28b0410f0..87b5c222be5 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -183,7 +183,12 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def if not refresh: package_index.clear() - for pkg_id in package_ids: + total_packages = len(package_ids) + for counter, pkg_id in enumerate(package_ids): + sys.stdout.write( + "\rIndexing dataset {0}/{1}".format(counter, total_packages) + ) + sys.stdout.flush() try: package_index.update_dict( logic.get_action('package_show')(context, From 2dbd53e59474a56186f85474c51c36710af2a1bf Mon Sep 17 00:00:00 2001 From: joetsoi Date: Tue, 15 Sep 2015 09:21:15 +0100 Subject: [PATCH 280/307] [#2461] Add custom translation directory after plugins --- ckan/lib/i18n.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ckan/lib/i18n.py b/ckan/lib/i18n.py index a869ecffc0b..033b86eeec9 100644 --- a/ckan/lib/i18n.py +++ b/ckan/lib/i18n.py @@ -140,6 +140,12 @@ def handle_request(request, tmpl_context): if lang != 'en': set_lang(lang) + + for plugin in PluginImplementations(ITranslation): + if lang in plugin.i18n_locales(): + _add_extra_translations(plugin.i18n_directory(), lang, + plugin.i18n_domain()) + extra_directory = config.get('ckan.i18n.extra_directory') extra_domain = config.get('ckan.i18n.extra_gettext_domain') extra_locales = aslist(config.get('ckan.i18n.extra_locales')) @@ -147,11 +153,6 @@ def handle_request(request, tmpl_context): if lang in extra_locales: _add_extra_translations(extra_directory, lang, extra_domain) - for plugin in PluginImplementations(ITranslation): - if lang in plugin.i18n_locales(): - _add_extra_translations(plugin.i18n_directory(), lang, - plugin.i18n_domain()) - tmpl_context.language = lang return lang From 052e3268a39faaa8b4fb06f0e7f90e6601b85245 Mon Sep 17 00:00:00 2001 From: joetsoi Date: Tue, 15 Sep 2015 09:26:02 +0100 Subject: [PATCH 281/307] [#2643] ITranslation docs --- ckan/pastertemplates/template/setup.cfg_tmpl | 21 +++ ckan/pastertemplates/template/setup.py_tmpl | 14 ++ ckan/plugins/interfaces.py | 2 +- ckanext/example_itranslation/plugin_v1.py | 9 ++ ckanext/example_itranslation/setup.cfg | 21 +++ doc/extensions/index.rst | 1 + doc/extensions/translating-extensions.rst | 160 +++++++++++++++++++ setup.py | 13 +- 8 files changed, 228 insertions(+), 13 deletions(-) create mode 100644 ckan/pastertemplates/template/setup.cfg_tmpl create mode 100644 ckanext/example_itranslation/plugin_v1.py create mode 100644 ckanext/example_itranslation/setup.cfg create mode 100644 doc/extensions/translating-extensions.rst diff --git a/ckan/pastertemplates/template/setup.cfg_tmpl b/ckan/pastertemplates/template/setup.cfg_tmpl new file mode 100644 index 00000000000..7b6b145410d --- /dev/null +++ b/ckan/pastertemplates/template/setup.cfg_tmpl @@ -0,0 +1,21 @@ +[extract_messages] +keywords = translate isPlural +add_comments = TRANSLATORS: +output_file = i18n/ckanext-{{ project_shortname }}.pot +width = 80 + +[init_catalog] +domain = ckanext-{{ project_shortname }} +input_file = i18n/ckanext-{{ project_shortname }}.pot +output_dir = i18n + +[update_catalog] +domain = ckanext-{{ project_shortname }} +input_file = i18n/ckanext-{{ project_shortname }}.pot +output_dir = i18n +previous = true + +[compile_catalog] +domain = ckanext-{{ project_shortname }} +directory = i18n +statistics = true diff --git a/ckan/pastertemplates/template/setup.py_tmpl b/ckan/pastertemplates/template/setup.py_tmpl index e1bc849280b..a2f6caf0dcc 100644 --- a/ckan/pastertemplates/template/setup.py_tmpl +++ b/ckan/pastertemplates/template/setup.py_tmpl @@ -79,6 +79,20 @@ setup( entry_points=''' [ckan.plugins] {{ project_shortname }}=ckanext.{{ project_shortname }}.plugin:{{ plugin_class_name }} + [babel.extractors] + ckan = ckan.lib.extract:extract_ckan ''', + + # If you are changing from the default layout of your extension, you may + # have to change the message extractors, you can read more about babel + # message extraction at + # http://babel.pocoo.org/docs/messages/#extraction-method-mapping-and-configuration + message_extractors={ + 'ckanext': [ + ('**.py', 'python', None), + ('**.js', 'javascript', None), + ('**/templates/**.html', 'ckan', None), + ], + } ) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index e51b6ce4ab8..d0b4ebb6ce3 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -1465,7 +1465,7 @@ def abort(self, status_code, detail, headers, comment): class ITranslation(Interface): def i18n_directory(self): - '''Change the directory of the *.mo translation files''' + '''Change the directory of the .mo translation files''' def i18n_locales(self): '''Change the list of locales that this plugin handles ''' diff --git a/ckanext/example_itranslation/plugin_v1.py b/ckanext/example_itranslation/plugin_v1.py new file mode 100644 index 00000000000..8ec304b0804 --- /dev/null +++ b/ckanext/example_itranslation/plugin_v1.py @@ -0,0 +1,9 @@ +from ckan import plugins +from ckan.plugins import toolkit + + +class ExampleITranslationPlugin(plugins.SingletonPlugin): + plugins.implements(plugins.IConfigurer) + + def update_config(self, config): + toolkit.add_template_directory(config, 'templates') diff --git a/ckanext/example_itranslation/setup.cfg b/ckanext/example_itranslation/setup.cfg new file mode 100644 index 00000000000..10f96302072 --- /dev/null +++ b/ckanext/example_itranslation/setup.cfg @@ -0,0 +1,21 @@ +[extract_messages] +keywords = translate isPlural +add_comments = TRANSLATORS: +output_file = i18n/ckanext-itranslation.pot +width = 80 + +[init_catalog] +domain = ckanext-itranslation +input_file = i18n/ckanext-itranslation.pot +output_dir = i18n + +[update_catalog] +domain = ckanext-itranslation +input_file = i18n/ckanext-itranslation.pot +output_dir = i18n +previous = true + +[compile_catalog] +domain = ckanext-itranslation +directory = i18n +statistics = true diff --git a/doc/extensions/index.rst b/doc/extensions/index.rst index 01cc6d95a72..aeecbff9706 100644 --- a/doc/extensions/index.rst +++ b/doc/extensions/index.rst @@ -38,3 +38,4 @@ features by developing your own CKAN extensions. plugin-interfaces plugins-toolkit validators + translating-extensions diff --git a/doc/extensions/translating-extensions.rst b/doc/extensions/translating-extensions.rst new file mode 100644 index 00000000000..c80af3539c1 --- /dev/null +++ b/doc/extensions/translating-extensions.rst @@ -0,0 +1,160 @@ +============================================= +Internationalizating of strings in extensions +============================================= + +.. seealso:: + + In order to internationalize you extension you must mark the strings for + internationalization. You can find out how to do this by reading + :doc: `/contributing/frontend/string-i18n.rst` + +.. seealso:: + + In this tutorial we are assuming that you have read the + :doc: `/extensions/tutorial` + +We will create a simple extension that demonstrates the translation of strings +inside extensions. After running + + paster --plugin=ckan create -t ckanext ckanext-itranslation + +Change and simply the ``plugin.py`` file to be + +.. literalinclude:: ../../ckanext/example_itranslation/plugin_v1.py + +Add a template file ``ckanext-itranslation/templates/home/index.html`` +containing + +.. literalinclude:: ../../ckanext/example_itranslation/templates/home/index.html + +This template just provides a sample string that we will be internationalizing +in this tutorial. + +------------------ +Extracting strings +------------------ + +.. tip:: + + If you have generated a new extension whilst following this tutorial the + default template will have generated these files for you and you can simply + run the ``extract_messages`` command immediately. + +Check your ``setup.py`` file in your extension for the following lines + +.. code-block:: python + :emphasize-lines: 5-6, 12-15 + + setup( + entry_points=''' + [ckan.plugins] + itranslation=ckanext.itranslation.plugin:ExampleITranslationPlugin + [babel.extractors] + ckan = ckan.lib.extract:extract_ckan + ''' + + message_extractors={ + 'ckanext': [ + ('**.py', 'python', None), + ('**.js', 'javascript', None), + ('**/templates/**.html', 'ckan', None), + ], + } + +These lines will already be present in our example, but if you are adding +internationalization to an older extension, you may need to add these them. +If you have your templates in a directory differing from the default location, +you may need to change the ``message_extractors`` stanza, you can read more +about message extractors at the `babel documentation `_ + + +Add an directory to store your translations + + mkdir ckanext-itranslations/i18n + +Next you will need a babel config file. Add ``setup.cfg`` file containing + +.. literalinclude:: ../../ckanext/example_itranslation/setup.cfg + +This file tells babel where the translation files are stored. +You can then run the ``extract_messages`` command to extract the strings from +your extension + + python setup.py extract_messages + +This will create a template PO file named +``ckanext/itranslations/i18n/ckanext-itranslation.pot`` +At this point, you can either upload an manage your translations using +transifex or manually create your translations. + +------------------------------ +Creating translations manually +------------------------------ + +We will be creating translation files for the ``fr`` locale. +Create the translation PO files for the locale that you are translating for +by running `init_catalog `_ + + python setup.py init_catalog -l fr + +This will generate a file called ``i18n/fr/LC_MESSAGES/ckanext-itranslation.po``. +Edit this file to contain the following. + +.. literalinclude:: ../../ckanext/example_itranslation/i18n/fr/LC_MESSAGES/ckanext-example_itranslation.po + :lines: 17-19 + + +--------------------------- +Translations with Transifex +--------------------------- + +Once you have created your translations, you can manage them using Transifex, +this is out side of the scope of this tutorial, but the Transifex documentation +provides tutorials on how to +`upload translations `_ +and how to manage them using them +`command line client `_ + + +--------------------- +Compiling the catalog +--------------------- + +Now compile the PO files by running + + python setup.py compile_catalog -l fr + +This will generate an mo file containing your translations. + +-------------------------- +The ITranslation interface +-------------------------- + +Once you have created the translated strings, you will need to inform CKAN that +your extension is translated by implementing the ``ITranslation`` interface in +your extension. Edit your ``plugin.py`` to contain the following. + +.. literalinclude:: ../../ckanext/example_itranslation/plugin.py + :emphasize-lines: 3, 6-7 + +Your done! To test your translated extension, make sure you add the extension to +your |development.ini| and run a ``paster serve`` and browse to +http://localhost:5000. You should find that switching to the ``fr`` locale in +the web interface should change the home page string to ``this is an itranslated +string`` + + +Advanced ITranslation usage +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you are translating a CKAN extension that already exists, or you have +structured your extension differently from the default layout. You may +have to tell CKAN where to locate your translated files, you can do this by +having your plugin not inherit from the ``DefaultTranslation`` class and +implement the ``ITranslation`` interface yourself. + +.. autosummary:: + + ~ckan.plugins.interfaces.ITranslation.i18n_directory + ~ckan.plugins.interfaces.ITranslation.i18n_locales + ~ckan.plugins.interfaces.ITranslation.i18n_domain diff --git a/setup.py b/setup.py index e69592511f3..bda1885d8b9 100644 --- a/setup.py +++ b/setup.py @@ -184,24 +184,13 @@ ('templates/importer/**', 'ignore', None), ('templates/**.html', 'ckan', None), ('templates_legacy/**.html', 'ckan', None), - ('ckan/templates/home/language.js', 'genshi', { - 'template_class': 'genshi.template:TextTemplate' - }), - ('templates/**.txt', 'genshi', { - 'template_class': 'genshi.template:TextTemplate' - }), - ('templates_legacy/**.txt', 'genshi', { - 'template_class': 'genshi.template:TextTemplate' - }), ('public/**', 'ignore', None), ], 'ckanext': [ ('**.py', 'python', None), + ('**.js', 'javascript', None), ('**.html', 'ckan', None), ('multilingual/solr/*.txt', 'ignore', None), - ('**.txt', 'genshi', { - 'template_class': 'genshi.template:TextTemplate' - }), ] }, entry_points=entry_points, From 35823d807c569626a151a7a938d7f681d13b6014 Mon Sep 17 00:00:00 2001 From: Denis Zgonjanin Date: Tue, 15 Sep 2015 10:50:17 -0400 Subject: [PATCH 282/307] add --quiet option to paster search index rebuild progress meter --- ckan/lib/cli.py | 23 ++++++++++++++--------- ckan/lib/search/__init__.py | 13 ++++++++----- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 1747eade27e..e98533cfed0 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -402,14 +402,14 @@ class SearchIndexCommand(CkanCommand): '''Creates a search index for all datasets Usage: - search-index [-i] [-o] [-r] [-e] rebuild [dataset_name] - reindex dataset_name if given, if not then rebuild - full search index (all datasets) - search-index rebuild_fast - reindex using multiprocessing using all cores. - This acts in the same way as rubuild -r [EXPERIMENTAL] - search-index check - checks for datasets not indexed - search-index show DATASET_NAME - shows index of a dataset - search-index clear [dataset_name] - clears the search index for the provided dataset or - for the whole ckan instance + search-index [-i] [-o] [-r] [-e] [-q] rebuild [dataset_name] - reindex dataset_name if given, if not then rebuild + full search index (all datasets) + search-index rebuild_fast - reindex using multiprocessing using all cores. + This acts in the same way as rubuild -r [EXPERIMENTAL] + search-index check - checks for datasets not indexed + search-index show DATASET_NAME - shows index of a dataset + search-index clear [dataset_name] - clears the search index for the provided dataset or + for the whole ckan instance ''' summary = __doc__.split('\n')[0] @@ -432,6 +432,10 @@ def __init__(self, name): action='store_true', default=False, help='Refresh current index (does not clear the existing one)') + self.parser.add_option('-q', '--quiet', dest='quiet', + action='store_true', default=False, + help='Do not output index rebuild progress') + self.parser.add_option('-e', '--commit-each', dest='commit_each', action='store_true', default=False, help= '''Perform a commit after indexing each dataset. This ensures that changes are @@ -474,7 +478,8 @@ def rebuild(self): rebuild(only_missing=self.options.only_missing, force=self.options.force, refresh=self.options.refresh, - defer_commit=(not self.options.commit_each)) + defer_commit=(not self.options.commit_each), + quiet=self.options.quiet) if not self.options.commit_each: commit() diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index 87b5c222be5..ba5e7aee81d 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -135,7 +135,8 @@ def notify(self, entity, operation): log.warn("Discarded Sync. indexing for: %s" % entity) -def rebuild(package_id=None, only_missing=False, force=False, refresh=False, defer_commit=False, package_ids=None): +def rebuild(package_id=None, only_missing=False, force=False, refresh=False, + defer_commit=False, package_ids=None, quiet=False): ''' Rebuilds the search index. @@ -185,10 +186,12 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def total_packages = len(package_ids) for counter, pkg_id in enumerate(package_ids): - sys.stdout.write( - "\rIndexing dataset {0}/{1}".format(counter, total_packages) - ) - sys.stdout.flush() + if not quiet: + sys.stdout.write( + "\rIndexing dataset {0}/{1}".format( + counter +1, total_packages) + ) + sys.stdout.flush() try: package_index.update_dict( logic.get_action('package_show')(context, From 02965b756a8e3af332c5066af92b649a75fe4521 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 15 Sep 2015 16:18:52 +0100 Subject: [PATCH 283/307] Clarify that site_url doesn't set the site's mount path In response to: http://stackoverflow.com/questions/32495459/install-ckan-in-a-sub-directory --- doc/maintaining/configuration.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index 46092f7afdd..4788dfde958 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -270,11 +270,15 @@ Example:: Default value: (an explicit value is mandatory) -The URL of your CKAN site. Many CKAN features that need an absolute URL to your +Set this to the URL of your CKAN site. Many CKAN features that need an absolute URL to your site use this setting. .. important:: It is mandatory to complete this setting +.. note:: If you want to mount CKAN at a path other than /, then this setting + should reflect that, but the URL you mount it at is determined by your + apache config (your WSGIScriptAlias path) (or equivalent for other servers). + .. warning:: This setting should not have a trailing / on the end. From a6d29b5a1639aafcfedfba32e5f4bdaec9c94b36 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 15 Sep 2015 18:09:12 +0100 Subject: [PATCH 284/307] Add missing import --- ckanext/datastore/tests/test_disable.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckanext/datastore/tests/test_disable.py b/ckanext/datastore/tests/test_disable.py index bc3a6f223b5..e47cfe4fc1c 100644 --- a/ckanext/datastore/tests/test_disable.py +++ b/ckanext/datastore/tests/test_disable.py @@ -1,3 +1,4 @@ +import nose import pylons.config as config import ckan.plugins as p From fd9bf893f9c0cfc77043f3b6384a62ee960612ac Mon Sep 17 00:00:00 2001 From: David Read Date: Wed, 16 Sep 2015 06:54:25 +0000 Subject: [PATCH 285/307] Fix test --- ckan/tests/legacy/lib/test_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/legacy/lib/test_cli.py b/ckan/tests/legacy/lib/test_cli.py index 3be780ba74f..cba14b4b1f5 100644 --- a/ckan/tests/legacy/lib/test_cli.py +++ b/ckan/tests/legacy/lib/test_cli.py @@ -80,7 +80,7 @@ def test_clear_and_rebuild_index(self): # Rebuild index self.search.args = () - self.search.options = FakeOptions(only_missing=False,force=False,refresh=False,commit_each=False) + self.search.options = FakeOptions(only_missing=False, force=False, refresh=False, commit_each=False, quiet=False) self.search.rebuild() pkg_count = model.Session.query(model.Package).filter(model.Package.state==u'active').count() From 6296d8813b4b9e174229c32075d16a9f42c024ca Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 22 Sep 2015 08:34:25 -0400 Subject: [PATCH 286/307] [#2636] comment for the scary stuff --- ckan/pastertemplates/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/pastertemplates/__init__.py b/ckan/pastertemplates/__init__.py index eea0ebe6923..8277b33e2b3 100644 --- a/ckan/pastertemplates/__init__.py +++ b/ckan/pastertemplates/__init__.py @@ -51,6 +51,8 @@ class CkanextTemplate(Template): def check_vars(self, vars, cmd): vars = Template.check_vars(self, vars, cmd) + # workaround for a paster issue https://github.com/ckan/ckan/issues/2636 + # this is only used from a short-lived paster command reload(sys) sys.setdefaultencoding('utf-8') From 5a287b04e475260356b83f73462bfad17347b052 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 22 Sep 2015 17:40:08 +0100 Subject: [PATCH 287/307] Separates search.clear() into two functions --- ckan/lib/cli.py | 9 ++++++--- ckan/lib/search/__init__.py | 15 +++++++++------ ckan/tests/helpers.py | 2 +- ckan/tests/legacy/__init__.py | 2 +- .../legacy/functional/api/model/test_tag.py | 12 ++++++------ .../legacy/functional/api/test_package_search.py | 16 ++++++++-------- ckan/tests/legacy/functional/test_group.py | 2 +- ckan/tests/legacy/lib/test_cli.py | 2 +- ckan/tests/legacy/lib/test_dictization.py | 2 +- .../tests/legacy/lib/test_solr_package_search.py | 10 +++++----- ...est_solr_package_search_synchronous_update.py | 12 ++++++------ ckan/tests/legacy/logic/test_action.py | 8 ++++---- ckan/tests/legacy/logic/test_tag.py | 2 +- ckan/tests/lib/dictization/test_model_dictize.py | 4 ++-- ckan/tests/logic/action/test_delete.py | 6 +++--- .../tests/test_example_idatasetform.py | 14 +++++++------- .../tests/test_multilingual_plugin.py | 2 +- 17 files changed, 63 insertions(+), 57 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index e98533cfed0..8845577f3d6 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -222,7 +222,7 @@ def command(self): os.remove(f) model.repo.clean_db() - search.clear() + search.clear_all() if self.verbose: print 'Cleaning DB: SUCCESS' elif cmd == 'upgrade': @@ -498,9 +498,12 @@ def show(self): pprint(index) def clear(self): - from ckan.lib.search import clear + from ckan.lib.search import clear, clear_all package_id = self.args[1] if len(self.args) > 1 else None - clear(package_id) + if not package_id: + clear_all() + else: + clear(package_id) def rebuild_fast(self): ### Get out config but without starting pylons environment #### diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index ba5e7aee81d..7e8d9fc7416 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -239,13 +239,16 @@ def show(package_reference): return package_query.get_index(package_reference) -def clear(package_reference=None): +def clear(package_reference): package_index = index_for(model.Package) - if package_reference: - log.debug("Clearing search index for dataset %s..." % - package_reference) - package_index.delete_package({'id': package_reference}) - elif not SIMPLE_SEARCH: + log.debug("Clearing search index for dataset %s..." % + package_reference) + package_index.delete_package({'id': package_reference}) + + +def clear_all(): + if not SIMPLE_SEARCH: + package_index = index_for(model.Package) log.debug("Clearing search index...") package_index.clear() diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 6d4fc4fbec1..e47c5c7fdc5 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -187,7 +187,7 @@ def _apply_config_changes(cls, cfg): def setup(self): '''Reset the database and clear the search indexes.''' reset_db() - search.clear() + search.clear_all() @classmethod def teardown_class(cls): diff --git a/ckan/tests/legacy/__init__.py b/ckan/tests/legacy/__init__.py index 982461a46bc..9005bf48ae0 100644 --- a/ckan/tests/legacy/__init__.py +++ b/ckan/tests/legacy/__init__.py @@ -320,7 +320,7 @@ def setup_test_search_index(): #from ckan import plugins if not is_search_supported(): raise SkipTest("Search not supported") - search.clear() + search.clear_all() #plugins.load('synchronous_search') def is_search_supported(): diff --git a/ckan/tests/legacy/functional/api/model/test_tag.py b/ckan/tests/legacy/functional/api/model/test_tag.py index 08e253f3734..7129b0ba5d5 100644 --- a/ckan/tests/legacy/functional/api/model/test_tag.py +++ b/ckan/tests/legacy/functional/api/model/test_tag.py @@ -1,20 +1,20 @@ import copy -from nose.tools import assert_equal +from nose.tools import assert_equal from ckan import model from ckan.lib.create_test_data import CreateTestData import ckan.lib.search as search from ckan.tests.legacy.functional.api.base import BaseModelApiTestCase -from ckan.tests.legacy.functional.api.base import Api1TestCase as Version1TestCase -from ckan.tests.legacy.functional.api.base import Api2TestCase as Version2TestCase +from ckan.tests.legacy.functional.api.base import Api1TestCase as Version1TestCase +from ckan.tests.legacy.functional.api.base import Api2TestCase as Version2TestCase class TagsTestCase(BaseModelApiTestCase): @classmethod def setup_class(cls): - search.clear() + search.clear_all() CreateTestData.create() cls.testsysadmin = model.User.by_name(u'testsysadmin') cls.comment = u'Comment umlaut: \xfc.' @@ -23,7 +23,7 @@ def setup_class(cls): @classmethod def teardown_class(cls): - search.clear() + search.clear_all() model.repo.rebuild_db() def test_register_get_ok(self): @@ -33,7 +33,7 @@ def test_register_get_ok(self): assert self.russian.name in results, results assert self.tolstoy.name in results, results assert self.flexible_tag.name in results, results - + def test_entity_get_ok(self): offset = self.tag_offset(self.russian.name) res = self.app.get(offset, status=self.STATUS_200_OK) diff --git a/ckan/tests/legacy/functional/api/test_package_search.py b/ckan/tests/legacy/functional/api/test_package_search.py index 512c5e6f5ce..15e4b343db2 100644 --- a/ckan/tests/legacy/functional/api/test_package_search.py +++ b/ckan/tests/legacy/functional/api/test_package_search.py @@ -34,7 +34,7 @@ def setup_class(self): @classmethod def teardown_class(cls): model.repo.rebuild_db() - search.clear() + search.clear_all() def assert_results(self, res_dict, expected_package_names): expected_pkgs = [self.package_ref_from_name(expected_package_name) \ @@ -62,7 +62,7 @@ def check(request_params, expected_params): def test_00_read_search_params_with_errors(self): def check_error(request_params): - assert_raises(ValueError, ApiController._get_search_params, request_params) + assert_raises(ValueError, ApiController._get_search_params, request_params) # uri json check_error(UnicodeMultiDict({'qjson': '{"q": illegal json}'})) # posted json @@ -109,7 +109,7 @@ def test_05_uri_json_tags(self): res_dict = self.data_from_res(res) self.assert_results(res_dict, [u'annakarenina']) assert res_dict['count'] == 1, res_dict - + def test_05_uri_json_tags_multiple(self): query = {'q': 'tags:russian tags:tolstoy'} json_query = self.dumps(query) @@ -131,7 +131,7 @@ def test_08_uri_qjson_malformed(self): offset = self.base_url + '?qjson="q":""' # user forgot the curly braces res = self.app.get(offset, status=400) self.assert_json_response(res, 'Bad request - Could not read parameters') - + def test_09_just_tags(self): offset = self.base_url + '?q=tags:russian' res = self.app.get(offset, status=200) @@ -199,7 +199,7 @@ def setup_class(self): @classmethod def teardown_class(cls): model.repo.rebuild_db() - search.clear() + search.clear_all() def test_07_uri_qjson_tags(self): query = {'q': '', 'tags':['tolstoy']} @@ -239,11 +239,11 @@ def test_07_uri_qjson_tags_reverse(self): assert res_dict['count'] == 2, res_dict def test_07_uri_qjson_extras(self): - # TODO: solr is not currently set up to allow partial matches + # TODO: solr is not currently set up to allow partial matches # and extras are not saved as multivalued so this # test will fail. Make extras multivalued or remove? raise SkipTest() - + query = {"geographic_coverage":"England"} json_query = self.dumps(query) offset = self.base_url + '?qjson=%s' % json_query @@ -267,7 +267,7 @@ def test_08_all_fields(self): rating=3.0) model.Session.add(rating) model.repo.commit_and_remove() - + query = {'q': 'russian', 'all_fields': 1} json_query = self.dumps(query) offset = self.base_url + '?qjson=%s' % json_query diff --git a/ckan/tests/legacy/functional/test_group.py b/ckan/tests/legacy/functional/test_group.py index 941d64d408f..632be1fd738 100644 --- a/ckan/tests/legacy/functional/test_group.py +++ b/ckan/tests/legacy/functional/test_group.py @@ -13,7 +13,7 @@ class TestGroup(FunctionalTestCase): @classmethod def setup_class(self): - search.clear() + search.clear_all() model.Session.remove() CreateTestData.create() diff --git a/ckan/tests/legacy/lib/test_cli.py b/ckan/tests/legacy/lib/test_cli.py index cba14b4b1f5..ce71774d658 100644 --- a/ckan/tests/legacy/lib/test_cli.py +++ b/ckan/tests/legacy/lib/test_cli.py @@ -8,7 +8,7 @@ from ckan.lib.create_test_data import CreateTestData from ckan.common import json -from ckan.lib.search import index_for,query_for +from ckan.lib.search import index_for,query_for, clear_all class TestDb: @classmethod diff --git a/ckan/tests/legacy/lib/test_dictization.py b/ckan/tests/legacy/lib/test_dictization.py index 8c17b1d6abd..89e61620ffd 100644 --- a/ckan/tests/legacy/lib/test_dictization.py +++ b/ckan/tests/legacy/lib/test_dictization.py @@ -31,7 +31,7 @@ class TestBasicDictize: def setup_class(cls): # clean the db so we can run these tests on their own model.repo.rebuild_db() - search.clear() + search.clear_all() CreateTestData.create() cls.package_expected = { diff --git a/ckan/tests/legacy/lib/test_solr_package_search.py b/ckan/tests/legacy/lib/test_solr_package_search.py index e19f5984221..86acdd765b4 100644 --- a/ckan/tests/legacy/lib/test_solr_package_search.py +++ b/ckan/tests/legacy/lib/test_solr_package_search.py @@ -56,7 +56,7 @@ def setup_class(cls): @classmethod def teardown_class(cls): model.repo.rebuild_db() - search.clear() + search.clear_all() def _pkg_names(self, result): return ' '.join(result['results']) @@ -316,7 +316,7 @@ def setup_class(cls): @classmethod def teardown_class(cls): model.repo.rebuild_db() - search.clear() + search.clear_all() def test_overall(self): check_search_results('annakarenina', 1, ['annakarenina']) @@ -358,7 +358,7 @@ def setup_class(cls): @classmethod def teardown_class(self): model.repo.rebuild_db() - search.clear() + search.clear_all() def _do_search(self, q, expected_pkgs, count=None): query = { @@ -422,7 +422,7 @@ def setup_class(cls): @classmethod def teardown_class(self): model.repo.rebuild_db() - search.clear() + search.clear_all() def _do_search(self, department, expected_pkgs, count=None): result = search.query_for(model.Package).run({'q': 'department: %s' % department}) @@ -467,7 +467,7 @@ def setup_class(cls): @classmethod def teardown_class(self): model.repo.rebuild_db() - search.clear() + search.clear_all() def _do_search(self, q, wanted_results): query = { diff --git a/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py b/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py index 9e8c50b1514..24f1e26f4f7 100644 --- a/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py +++ b/ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py @@ -52,19 +52,19 @@ def setup_class(cls): @classmethod def teardown_class(cls): model.repo.rebuild_db() - search.clear() + search.clear_all() def setup(self): self._create_package() - + def teardown(self): self._remove_package() self._remove_package(u'new_name') - + def _create_package(self, package=None): CreateTestData.create_arbitrary(self.new_pkg_dict) return model.Package.by_name(self.new_pkg_dict['name']) - + def _remove_package(self, name=None): package = model.Package.by_name(name or 'council-owned-litter-bins') if package: @@ -84,7 +84,7 @@ def test_03_update_package_from_dict(self): extra = model.PackageExtra(key='published_by', value='barrow') package._extras[extra.key] = extra model.repo.commit_and_remove() - + check_search_results('', 3) check_search_results('barrow', 1, ['new_name']) @@ -106,5 +106,5 @@ def test_04_delete_package_from_dict(self): rev = model.repo.new_revision() package.delete() model.repo.commit_and_remove() - + check_search_results('', 2) diff --git a/ckan/tests/legacy/logic/test_action.py b/ckan/tests/legacy/logic/test_action.py index 156c80f70ca..05bc73ba9e1 100644 --- a/ckan/tests/legacy/logic/test_action.py +++ b/ckan/tests/legacy/logic/test_action.py @@ -35,7 +35,7 @@ class TestAction(WsgiAppCase): @classmethod def setup_class(cls): model.repo.rebuild_db() - search.clear() + search.clear_all() CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') @@ -1349,7 +1349,7 @@ class TestBulkActions(WsgiAppCase): @classmethod def setup_class(cls): - search.clear() + search.clear_all() model.Session.add_all([ model.User(name=u'sysadmin', apikey=u'sysadmin', password=u'sysadmin', sysadmin=True), @@ -1436,7 +1436,7 @@ class TestResourceAction(WsgiAppCase): @classmethod def setup_class(cls): - search.clear() + search.clear_all() CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') @@ -1539,7 +1539,7 @@ class TestRelatedAction(WsgiAppCase): @classmethod def setup_class(cls): - search.clear() + search.clear_all() CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') diff --git a/ckan/tests/legacy/logic/test_tag.py b/ckan/tests/legacy/logic/test_tag.py index 9d9d8667265..0419f2ffc90 100644 --- a/ckan/tests/legacy/logic/test_tag.py +++ b/ckan/tests/legacy/logic/test_tag.py @@ -10,7 +10,7 @@ class TestAction(WsgiAppCase): @classmethod def setup_class(cls): - search.clear() + search.clear_all() CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') diff --git a/ckan/tests/lib/dictization/test_model_dictize.py b/ckan/tests/lib/dictization/test_model_dictize.py index 493104980cb..0c8dabe128d 100644 --- a/ckan/tests/lib/dictization/test_model_dictize.py +++ b/ckan/tests/lib/dictization/test_model_dictize.py @@ -14,7 +14,7 @@ class TestGroupListDictize: def setup(self): helpers.reset_db() - search.clear() + search.clear_all() def test_group_list_dictize(self): group = factories.Group() @@ -136,7 +136,7 @@ class TestGroupDictize: def setup(self): helpers.reset_db() - search.clear() + search.clear_all() def test_group_dictize(self): group = factories.Group(name='test_dictize') diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 9c753867d83..ab2b25b9539 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -186,7 +186,7 @@ def test_dataset_in_a_purged_group_no_longer_shows_that_group(self): assert_equals(dataset_shown['groups'], []) def test_purged_group_is_not_in_search_results_for_its_ex_dataset(self): - search.clear() + search.clear_all() group = factories.Group() dataset = factories.Dataset(groups=[{'name': group['name']}]) @@ -288,7 +288,7 @@ def test_dataset_in_a_purged_org_no_longer_shows_that_org(self): assert_equals(dataset_shown['owner_org'], None) def test_purged_org_is_not_in_search_results_for_its_ex_dataset(self): - search.clear() + search.clear_all() org = factories.Organization() dataset = factories.Dataset(owner_org=org['id']) @@ -394,7 +394,7 @@ def test_group_no_longer_shows_its_purged_dataset(self): assert_equals(dataset_shown['packages'], []) def test_purged_dataset_is_not_in_search_results(self): - search.clear() + search.clear_all() dataset = factories.Dataset() def get_search_results(): diff --git a/ckanext/example_idatasetform/tests/test_example_idatasetform.py b/ckanext/example_idatasetform/tests/test_example_idatasetform.py index b437716066f..99f5c79f94c 100644 --- a/ckanext/example_idatasetform/tests/test_example_idatasetform.py +++ b/ckanext/example_idatasetform/tests/test_example_idatasetform.py @@ -18,13 +18,13 @@ def setup_class(cls): def teardown(self): model.repo.rebuild_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() @classmethod def teardown_class(cls): helpers.reset_db() model.repo.rebuild_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() config.clear() config.update(cls.original_config) @@ -97,7 +97,7 @@ def teardown(self): def teardown_class(cls): plugins.unload('example_idatasetform_v4') helpers.reset_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() config.clear() config.update(cls.original_config) @@ -139,13 +139,13 @@ def setup_class(cls): def teardown(self): model.repo.rebuild_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() @classmethod def teardown_class(cls): plugins.unload('example_idatasetform') helpers.reset_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() config.clear() config.update(cls.original_config) @@ -212,13 +212,13 @@ def setup_class(cls): def teardown(self): model.repo.rebuild_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() @classmethod def teardown_class(cls): plugins.unload('example_idatasetform') helpers.reset_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() config.clear() config.update(cls.original_config) diff --git a/ckanext/multilingual/tests/test_multilingual_plugin.py b/ckanext/multilingual/tests/test_multilingual_plugin.py index ab795084667..118ad195470 100644 --- a/ckanext/multilingual/tests/test_multilingual_plugin.py +++ b/ckanext/multilingual/tests/test_multilingual_plugin.py @@ -58,7 +58,7 @@ def teardown(cls): ckan.plugins.unload('multilingual_group') ckan.plugins.unload('multilingual_tag') ckan.model.repo.rebuild_db() - ckan.lib.search.clear() + ckan.lib.search.clear_all() def test_user_read_translation(self): '''Test the translation of datasets on user view pages by the From 2c5e3ce87fff1474abcc752bc820085dfef8eec4 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Wed, 23 Sep 2015 17:27:34 +0100 Subject: [PATCH 288/307] [#2654] Add Pylons ungettext to toolkit --- ckan/plugins/toolkit.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 2e95a6f9d57..d61705ec947 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -18,6 +18,7 @@ class _Toolkit(object): contents = [ ## Imported functions/objects ## '_', # i18n translation + 'ungettext', # i18n translation (plural forms) 'c', # template context 'request', # http request object 'render', # template render function @@ -111,6 +112,19 @@ def _initialize(self): msg = toolkit._("Hello") +''' + t['ungettext'] = common.ungettext + self.docstring_overrides['ungettext'] = '''The Pylons ``ungettext`` + function. + +Mark a string for translation that has pural forms in the format +``ungettext(singular, plural, n)``. Returns the localized unicode string of +the pluralized value. + +Mark a string to be localized as follows:: + + msg = toolkit.ungettext("Mouse", "Mice", len(mouses)) + ''' t['c'] = common.c self.docstring_overrides['c'] = '''The Pylons template context object. From 3dad5d1af5633fb2de57c5730a2bdaa062e5bedd Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 29 Sep 2015 21:18:13 -0400 Subject: [PATCH 289/307] [#2382] fix tests --- ckan/tests/legacy/functional/api/model/test_group.py | 2 +- ckan/tests/legacy/lib/test_dictization_schema.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/tests/legacy/functional/api/model/test_group.py b/ckan/tests/legacy/functional/api/model/test_group.py index dbfb8fe8b54..94be9e1ab68 100644 --- a/ckan/tests/legacy/functional/api/model/test_group.py +++ b/ckan/tests/legacy/functional/api/model/test_group.py @@ -15,7 +15,7 @@ class GroupsTestCase(BaseModelApiTestCase): @classmethod def setup_class(cls): - search.clear() + search.clear_all() CreateTestData.create() cls.user_name = u'russianfan' # created in CreateTestData cls.init_extra_environ(cls.user_name) diff --git a/ckan/tests/legacy/lib/test_dictization_schema.py b/ckan/tests/legacy/lib/test_dictization_schema.py index 864f187be3f..c3114cc3cb3 100644 --- a/ckan/tests/legacy/lib/test_dictization_schema.py +++ b/ckan/tests/legacy/lib/test_dictization_schema.py @@ -19,7 +19,7 @@ def setup(self): @classmethod def setup_class(cls): - search.clear() + search.clear_all() CreateTestData.create() @classmethod From b847f7fd0b5f4fd33e66078e455a40716ec440fb Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 6 Oct 2015 14:21:23 +0100 Subject: [PATCH 290/307] [#2669] Fix title munge resulting in multiple dashes. --- ckan/lib/munge.py | 2 +- ckan/tests/lib/test_munge.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py index 484e0f5b0df..c486e6ef9bf 100644 --- a/ckan/lib/munge.py +++ b/ckan/lib/munge.py @@ -34,7 +34,7 @@ def munge_title_to_name(name): # take out not-allowed characters name = re.sub('[^a-zA-Z0-9-_]', '', name).lower() # remove doubles - name = re.sub('--', '-', name) + name = re.sub('-+', '-', name) # remove leading or trailing hyphens name = name.strip('-') # if longer than max_length, keep last word if a year diff --git a/ckan/tests/lib/test_munge.py b/ckan/tests/lib/test_munge.py index de3b9e8051b..6cbf688e87d 100644 --- a/ckan/tests/lib/test_munge.py +++ b/ckan/tests/lib/test_munge.py @@ -104,14 +104,14 @@ class TestMungeTitleToName(object): # (original, expected) munge_list = [ ('unchanged', 'unchanged'), - ('some spaces here', 'some-spaces-here'), + ('some spaces here &here', 'some-spaces-here-here'), ('s', 's_'), # too short ('random:other%character&', 'random-othercharacter'), (u'u with umlaut \xfc', 'u-with-umlaut-u'), ('reallylong' * 12, 'reallylong' * 9 + 'reall'), ('reallylong' * 12 + ' - 2012', 'reallylong' * 9 + '-2012'), ('10cm - 50cm Near InfraRed (NI) Digital Aerial Photography (AfA142)', - '10cm--50cm-near-infrared-ni-digital-aerial-photography-afa142') + '10cm-50cm-near-infrared-ni-digital-aerial-photography-afa142') ] def test_munge_title_to_name(self): @@ -128,7 +128,8 @@ class TestMungeTag: ('unchanged', 'unchanged'), ('s', 's_'), # too short ('some spaces here', 'some-spaces--here'), - ('random:other%character&', 'randomothercharacter') + ('random:other%characters&_.here', 'randomothercharactershere'), + ('river-water-dashes', 'river-water-dashes'), ] def test_munge_tag(self): @@ -137,7 +138,7 @@ def test_munge_tag(self): munge = munge_tag(org) nose_tools.assert_equal(munge, exp) - def test_munge_tag_muliple_pass(self): + def test_munge_tag_multiple_pass(self): '''Munge a list of tags muliple times gives expected results.''' for org, exp in self.munge_list: first_munge = munge_tag(org) From ff86a146a8253191c85f01c87fc69b0a33a9e748 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 6 Oct 2015 14:27:36 +0100 Subject: [PATCH 291/307] Document issue policy --- doc/contributing/issues.rst | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/doc/contributing/issues.rst b/doc/contributing/issues.rst index 1d027321c13..ab014b595d5 100644 --- a/doc/contributing/issues.rst +++ b/doc/contributing/issues.rst @@ -10,6 +10,33 @@ searching first to see if there's already an issue for your bug). If you can fix the bug yourself, please :doc:`send a pull request `! -.. todo:: +Do not use an issue to ask how to do something - for that use StackOverflow with the 'ckan' tag. + +Do not use an issue to suggest an significant change to CKAN - instead create an issue at https://github.com/ckan/ideas-and-roadmap. + + +Writing a good issue +==================== + +* Describe what went wrong +* Say what you were doing when it went wrong +* If in doubt, provide detailed steps for someone else to recreate the problem. +* A screenshot is often helpful +* If it is a 500 error / ServerError / exception then it's essential to supply the full stack trace provided in the CKAN log. + +Issues process +============== + +The CKAN Technical Team reviews new issues twice a week. They aim to assign someone on the Team to take responsibility for it. These are the sorts of actions to expect: + +* If it is a serious bug and the person who raised it won't fix it then the Technical Team will aim to create a fix. + +* A feature that you plan to code shortly will be happily discussed. It's often good to get the team's support for a feature before writing lots of code. You can then quote the issue number in the commit messages and branch name. (Larger changes or suggestions by non-contributers are better discussed on https://github.com/ckan/ideas-and-roadmap instead) + +* Features may be marked "Good for Contribution" which means the Team is happy to see this happen, but the Team are not offering to do it. + +Old issues +========== + +If an issue has little activity for 12 months then it should be closed. If someone is still keen for it to happen then they can comment, re-open it and push it forward. - Could put more detail here about how to make a good bug report. From 77ec5cbec151c9feb84c218efec7d12510b02e6c Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 6 Oct 2015 14:31:53 +0100 Subject: [PATCH 292/307] Issue policy - formatting --- doc/contributing/issues.rst | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/doc/contributing/issues.rst b/doc/contributing/issues.rst index ab014b595d5..6f8f7bfa6db 100644 --- a/doc/contributing/issues.rst +++ b/doc/contributing/issues.rst @@ -10,9 +10,11 @@ searching first to see if there's already an issue for your bug). If you can fix the bug yourself, please :doc:`send a pull request `! -Do not use an issue to ask how to do something - for that use StackOverflow with the 'ckan' tag. +Do not use an issue to ask how to do something - for that use StackOverflow +with the 'ckan' tag. -Do not use an issue to suggest an significant change to CKAN - instead create an issue at https://github.com/ckan/ideas-and-roadmap. +Do not use an issue to suggest an significant change to CKAN - instead create +an issue at https://github.com/ckan/ideas-and-roadmap. Writing a good issue @@ -22,21 +24,31 @@ Writing a good issue * Say what you were doing when it went wrong * If in doubt, provide detailed steps for someone else to recreate the problem. * A screenshot is often helpful -* If it is a 500 error / ServerError / exception then it's essential to supply the full stack trace provided in the CKAN log. +* If it is a 500 error / ServerError / exception then it's essential to supply + the full stack trace provided in the CKAN log. Issues process ============== -The CKAN Technical Team reviews new issues twice a week. They aim to assign someone on the Team to take responsibility for it. These are the sorts of actions to expect: +The CKAN Technical Team reviews new issues twice a week. They aim to assign +someone on the Team to take responsibility for it. These are the sorts of +actions to expect: -* If it is a serious bug and the person who raised it won't fix it then the Technical Team will aim to create a fix. +* If it is a serious bug and the person who raised it won't fix it then the + Technical Team will aim to create a fix. -* A feature that you plan to code shortly will be happily discussed. It's often good to get the team's support for a feature before writing lots of code. You can then quote the issue number in the commit messages and branch name. (Larger changes or suggestions by non-contributers are better discussed on https://github.com/ckan/ideas-and-roadmap instead) +* A feature that you plan to code shortly will be happily discussed. It's often + good to get the team's support for a feature before writing lots of code. You + can then quote the issue number in the commit messages and branch name. + (Larger changes or suggestions by non-contributers are better discussed on + https://github.com/ckan/ideas-and-roadmap instead) -* Features may be marked "Good for Contribution" which means the Team is happy to see this happen, but the Team are not offering to do it. +* Features may be marked "Good for Contribution" which means the Team is happy + to see this happen, but the Team are not offering to do it. Old issues ========== -If an issue has little activity for 12 months then it should be closed. If someone is still keen for it to happen then they can comment, re-open it and push it forward. - +If an issue has little activity for 12 months then it should be closed. If +someone is still keen for it to happen then they should comment, re-open it and +push it forward. From c9b7104f95af98ccca0fdfacb0b3eaf0777421de Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 6 Oct 2015 14:32:39 +0100 Subject: [PATCH 293/307] Document what the options are for CKAN icons. --- ckan/config/routing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/config/routing.py b/ckan/config/routing.py index 2350b9cccd5..15b904666e5 100644 --- a/ckan/config/routing.py +++ b/ckan/config/routing.py @@ -31,7 +31,8 @@ def connect(self, *args, **kw): Also takes some additional params: :param ckan_icon: name of the icon to be associated with this route, - e.g. 'group', 'time' + e.g. 'group', 'time'. Available icons are listed here: + http://fortawesome.github.io/Font-Awesome/3.2.1/icons/ :type ckan_icon: string :param highlight_actions: space-separated list of controller actions that should be treated as the same as this named route for menu From d82cfb0e4494cdeda1231cff725e1ff29844c6af Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 9 Oct 2015 15:34:51 +0100 Subject: [PATCH 294/307] [#2470] Upgrade requests version pin. --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index d395aa423a2..e0158166eca 100644 --- a/requirements.in +++ b/requirements.in @@ -18,7 +18,7 @@ python-dateutil>=1.5.0,<2.0.0 pyutilib.component.core==4.5.3 repoze.who-friendlyform==1.0.8 repoze.who==2.0 -requests==2.3.0 +requests==2.7.0 Routes==1.13 solrpy==0.9.5 sqlalchemy-migrate==0.9.1 diff --git a/requirements.txt b/requirements.txt index 7e3043640a1..cb693e702ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -31,7 +31,7 @@ pyutilib.component.core==4.5.3 repoze.lru==0.6 repoze.who==2.0 repoze.who-friendlyform==1.0.8 -requests==2.3.0 +requests==2.7.0 simplejson==3.3.1 six==1.7.3 solrpy==0.9.5 From 50a98caf02ede59c0f7c6cf7e4aa738424ee49c4 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 9 Oct 2015 15:43:12 +0000 Subject: [PATCH 295/307] [#2561] Run pip-compile again --- requirements.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 07a1c25596d..e899de45960 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,15 +4,15 @@ # # pip-compile requirements.in # -argparse==1.3.0 # via ofs +argparse==1.4.0 # via ofs babel==0.9.6 beaker==1.7.0 -decorator==4.0.2 # via pylons, sqlalchemy-migrate +decorator==4.0.4 # via pylons, sqlalchemy-migrate fanstatic==0.12 formencode==1.3.0 # via pylons Genshi==0.6 Jinja2==2.6 -mako==1.0.1 # via pylons +mako==1.0.2 # via pylons markupsafe==0.23 # via mako, webhelpers nose==1.3.7 # via pylons ofs==0.4.1 @@ -33,13 +33,13 @@ repoze.who==2.0 requests==2.3.0 routes==1.13 simplejson==3.8.0 # via pylons -six==1.9.0 # via pastescript, sqlalchemy-migrate +six==1.10.0 # via pastescript, sqlalchemy-migrate solrpy==0.9.5 sqlalchemy-migrate==0.9.1 sqlalchemy==0.9.6 sqlparse==0.1.11 tempita==0.5.2 # via pylons, sqlalchemy-migrate, weberror -unicodecsv==0.13.0 +unicodecsv==0.14.1 vdm==0.13 weberror==0.11 # via pylons webhelpers==1.3 @@ -49,5 +49,5 @@ zope.interface==4.1.1 # The following packages are commented out because they are # considered to be unsafe in a requirements file: -# pip==7.1.0 # via pbr -# setuptools==18.0.1 +# pip==7.1.2 # via pbr +# setuptools==18.3.2 From 301efbaa8040af7e9c75998f8e961aa51ac3fd73 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 9 Oct 2015 17:12:15 +0000 Subject: [PATCH 296/307] [#2561] Add test to show issue with lazy json and the newer simplejson library. --- ckan/lib/lazyjson.py | 9 ++++++++- ckan/logic/action/get.py | 2 +- ckan/tests/lib/test_lazyjson.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 ckan/tests/lib/test_lazyjson.py diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index c4c29160b49..67d8772402e 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -3,7 +3,14 @@ class LazyJSONObject(dict): - '''An object that behaves like a dict returned from json.loads''' + '''An object that behaves like a dict returned from json.loads, + however it will not actually do the expensive decoding from a JSON string + into a dict unless you start treating it like a dict. + + This is therefore useful for the situation where there's a good chance you + won't need to use the data in dict form, and all you're going to do is + json.dumps it again, for which your original string is returned. + ''' def __init__(self, json_string): self._json_string = json_string self._json_dict = None diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 16ddb29a494..a48df52d21a 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1056,7 +1056,7 @@ def package_show(context, data_dict): package_dict_validated = False metadata_modified = pkg.metadata_modified.isoformat() search_metadata_modified = search_result['metadata_modified'] - # solr stores less precice datetime, + # solr stores less precise datetime, # truncate to 22 charactors to get good enough match if metadata_modified[:22] != search_metadata_modified[:22]: package_dict = None diff --git a/ckan/tests/lib/test_lazyjson.py b/ckan/tests/lib/test_lazyjson.py new file mode 100644 index 00000000000..547e7a6b4ec --- /dev/null +++ b/ckan/tests/lib/test_lazyjson.py @@ -0,0 +1,28 @@ +from nose.tools import assert_equal + +from ckan.lib.lazyjson import LazyJSONObject +import ckan.lib.helpers as h + + +class TestLazyJson(object): + def test_dump_without_necessarily_going_via_a_dict(self): + json_string = '{"title": "test_2"}' + lazy_json_obj = LazyJSONObject(json_string) + dumped = h.json.dumps( + lazy_json_obj, + for_json=True) + assert_equal(dumped, json_string) + + def test_dump_without_needing_to_go_via_a_dict(self): + json_string = '"invalid" JSON to [{}] ensure it doesnt become a dict' + lazy_json_obj = LazyJSONObject(json_string) + dumped = h.json.dumps( + lazy_json_obj, + for_json=True) + assert_equal(dumped, json_string) + + def test_treat_like_a_dict(self): + json_string = '{"title": "test_2"}' + lazy_json_obj = LazyJSONObject(json_string) + assert_equal(lazy_json_obj.keys(), ['title']) + assert_equal(len(lazy_json_obj), 1) From 7cd6cc13cb570b62748bb7c885155561ed7b8edd Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 12 Oct 2015 14:28:44 +0000 Subject: [PATCH 297/307] [#2561] Use old version of simplejson for now cos of #2681. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 478b294526e..c0aef876f31 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ repoze.who-friendlyform==1.0.8 repoze.who==2.0 requests==2.3.0 routes==1.13 -simplejson==3.8.0 # via pylons +simplejson==3.3.1 # via pylons HAND-FIXED FOR NOW #2681 six==1.10.0 # via pastescript, sqlalchemy-migrate solrpy==0.9.5 sqlalchemy-migrate==0.9.1 From ab46d45f932cfb8062c3d1e40bb402ee193b9578 Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 12 Oct 2015 16:25:55 +0100 Subject: [PATCH 298/307] Revert "[#2561] Add test to show issue with lazy json and the newer simplejson library." This reverts commit 301efbaa8040af7e9c75998f8e961aa51ac3fd73. --- ckan/lib/lazyjson.py | 9 +-------- ckan/logic/action/get.py | 2 +- ckan/tests/lib/test_lazyjson.py | 28 ---------------------------- 3 files changed, 2 insertions(+), 37 deletions(-) delete mode 100644 ckan/tests/lib/test_lazyjson.py diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py index 67d8772402e..c4c29160b49 100644 --- a/ckan/lib/lazyjson.py +++ b/ckan/lib/lazyjson.py @@ -3,14 +3,7 @@ class LazyJSONObject(dict): - '''An object that behaves like a dict returned from json.loads, - however it will not actually do the expensive decoding from a JSON string - into a dict unless you start treating it like a dict. - - This is therefore useful for the situation where there's a good chance you - won't need to use the data in dict form, and all you're going to do is - json.dumps it again, for which your original string is returned. - ''' + '''An object that behaves like a dict returned from json.loads''' def __init__(self, json_string): self._json_string = json_string self._json_dict = None diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a48df52d21a..16ddb29a494 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1056,7 +1056,7 @@ def package_show(context, data_dict): package_dict_validated = False metadata_modified = pkg.metadata_modified.isoformat() search_metadata_modified = search_result['metadata_modified'] - # solr stores less precise datetime, + # solr stores less precice datetime, # truncate to 22 charactors to get good enough match if metadata_modified[:22] != search_metadata_modified[:22]: package_dict = None diff --git a/ckan/tests/lib/test_lazyjson.py b/ckan/tests/lib/test_lazyjson.py deleted file mode 100644 index 547e7a6b4ec..00000000000 --- a/ckan/tests/lib/test_lazyjson.py +++ /dev/null @@ -1,28 +0,0 @@ -from nose.tools import assert_equal - -from ckan.lib.lazyjson import LazyJSONObject -import ckan.lib.helpers as h - - -class TestLazyJson(object): - def test_dump_without_necessarily_going_via_a_dict(self): - json_string = '{"title": "test_2"}' - lazy_json_obj = LazyJSONObject(json_string) - dumped = h.json.dumps( - lazy_json_obj, - for_json=True) - assert_equal(dumped, json_string) - - def test_dump_without_needing_to_go_via_a_dict(self): - json_string = '"invalid" JSON to [{}] ensure it doesnt become a dict' - lazy_json_obj = LazyJSONObject(json_string) - dumped = h.json.dumps( - lazy_json_obj, - for_json=True) - assert_equal(dumped, json_string) - - def test_treat_like_a_dict(self): - json_string = '{"title": "test_2"}' - lazy_json_obj = LazyJSONObject(json_string) - assert_equal(lazy_json_obj.keys(), ['title']) - assert_equal(len(lazy_json_obj), 1) From 8465a72759d5219d99a75b5cfb16cf7f77fe33c4 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 15 Oct 2015 11:12:26 +0100 Subject: [PATCH 299/307] [#2589] Document hard limit on package_search --- ckan/logic/action/get.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 16ddb29a494..242edfbf6c9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1768,7 +1768,8 @@ def package_search(context, data_dict): documentation, this is a comma-separated string of field names and sort-orderings. :type sort: string - :param rows: the number of matching rows to return. + :param rows: the number of matching rows to return. There is a hard limit + of 1000 datasets per query. :type rows: int :param start: the offset in the complete result for where the set of returned datasets should begin. From b33a08d3c3d1afc9daccdfe149622950326c388d Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 20 Oct 2015 21:07:19 -0400 Subject: [PATCH 300/307] [#2696] errors block for resource_form to match package_form --- ckan/templates/package/snippets/resource_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/snippets/resource_form.html b/ckan/templates/package/snippets/resource_form.html index 3946f1e8c81..bd33bcc28e3 100644 --- a/ckan/templates/package/snippets/resource_form.html +++ b/ckan/templates/package/snippets/resource_form.html @@ -12,7 +12,7 @@ {% endif %} {% endblock %} - {{ form.errors(error_summary) }} + {% block errors %}{{ form.errors(error_summary) }}{% endblock %} From b6d6a25d0183b9eb35b598ea9a51c7da2c096401 Mon Sep 17 00:00:00 2001 From: Stefan Oderbolz Date: Wed, 21 Oct 2015 10:41:29 +0200 Subject: [PATCH 301/307] Make sure package_autocomplete uses LIKE with wildcards The current implementation uses 'q%' to search for packages starting with a certain search term. This commit changes this to use the following format: '%q%', thus search for the term anywhere in the title and name field. --- ckan/logic/action/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 242edfbf6c9..e6c2f30f33f 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1608,7 +1608,7 @@ def package_autocomplete(context, data_dict): limit = data_dict.get('limit', 10) q = data_dict['q'] - like_q = u"%s%%" % q + like_q = u"%%%s%%" % q query = model.Session.query(model.Package) query = query.filter(model.Package.state == 'active') From a843385bf1963c6ee26aca70f98bb871b97abfe8 Mon Sep 17 00:00:00 2001 From: Mark Winterbottom Date: Wed, 21 Oct 2015 16:23:22 +0100 Subject: [PATCH 302/307] Fixed typo. --- doc/extensions/adding-custom-fields.rst | 82 ++++++++++++------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/doc/extensions/adding-custom-fields.rst b/doc/extensions/adding-custom-fields.rst index b71b7627a41..f66f2f46372 100644 --- a/doc/extensions/adding-custom-fields.rst +++ b/doc/extensions/adding-custom-fields.rst @@ -36,7 +36,7 @@ of available validators can be found at the :doc:`validators`. You can also define your own :ref:`custom-validators`. We will be customizing these schemas to add our additional fields. The -:py:class:`~ckan.plugins.interfaces.IDatasetForm` interface allows us to +:py:class:`~ckan.plugins.interfaces.IDatasetForm` interface allows us to override the schemas for creation, updating and displaying of datasets. .. autosummary:: @@ -49,7 +49,7 @@ override the schemas for creation, updating and displaying of datasets. CKAN allows you to have multiple IDatasetForm plugins, each handling different dataset types. So you could customize the CKAN web front end, for different -types of datasets. In this tutorial we will be defining our plugin as the +types of datasets. In this tutorial we will be defining our plugin as the fallback plugin. This plugin is used if no other IDatasetForm plugin is found that handles that dataset type. @@ -61,9 +61,9 @@ Adding custom fields to datasets ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Create a new plugin named ``ckanext-extrafields`` and create a class named -``ExampleIDatasetFormPlugins`` inside -``ckanext-extrafields/ckanext/extrafields/plugins.py`` that implements the -``IDatasetForm`` interface and inherits from ``SingletonPlugin`` and +``ExampleIDatasetFormPlugins`` inside +``ckanext-extrafields/ckanext/extrafields/plugins.py`` that implements the +``IDatasetForm`` interface and inherits from ``SingletonPlugin`` and ``DefaultDatasetForm``. .. literalinclude:: ../../ckanext/example_idatasetform/plugin_v1.py @@ -73,10 +73,10 @@ Updating the CKAN schema ^^^^^^^^^^^^^^^^^^^^^^^^ The :py:meth:`~ckan.plugins.interfaces.IDatasetForm.create_package_schema` -function is used whenever a new dataset is created, we'll want update the +function is used whenever a new dataset is created, we'll want update the default schema and insert our custom field here. We will fetch the default -schema defined in -:py:func:`~ckan.logic.schema.default_create_package_schema` by running +schema defined in +:py:func:`~ckan.logic.schema.default_create_package_schema` by running :py:meth:`~ckan.plugins.interfaces.IDatasetForm.create_package_schema`'s super function and update it. @@ -108,10 +108,10 @@ converted *from* an extras field. So we want to use the Dataset types ^^^^^^^^^^^^^ -The :py:meth:`~ckan.plugins.interfaces.IDatasetForm.package_types` function -defines a list of dataset types that this plugin handles. Each dataset has a -field containing its type. Plugins can register to handle specific types of -dataset and ignore others. Since our plugin is not for any specific type of +The :py:meth:`~ckan.plugins.interfaces.IDatasetForm.package_types` function +defines a list of dataset types that this plugin handles. Each dataset has a +field containing its type. Plugins can register to handle specific types of +dataset and ignore others. Since our plugin is not for any specific type of dataset and we want our plugin to be the default handler, we update the plugin code to contain the following: @@ -130,26 +130,26 @@ IConfigurer interface :start-after: import ckan.plugins.toolkit as tk :end-before: def create_package_schema(self): -This interface allows to implement a function +This interface allows to implement a function :py:meth:`~ckan.plugins.interfaces.IDatasetForm.update_config` that allows us to update the CKAN config, in our case we want to add an additional location -for CKAN to look for templates. Add the following code to your plugin. +for CKAN to look for templates. Add the following code to your plugin. .. literalinclude:: ../../ckanext/example_idatasetform/plugin_v2.py :pyobject: ExampleIDatasetFormPlugin.update_config You will also need to add a directory under your extension directory to store -the templates. Create a directory called +the templates. Create a directory called ``ckanext-extrafields/ckanext/extrafields/templates/`` and the subdirectories ``ckanext-extrafields/ckanext/extrafields/templates/package/snippets/``. We need to override a few templates in order to get our custom field rendered. A common option when using a custom schema is to remove the default custom field handling that allows arbitrary key/value pairs. Create a template -file in our templates directory called +file in our templates directory called ``package/snippets/package_metadata_fields.html`` containing - + .. literalinclude:: ../../ckanext/example_idatasetform/templates/package/snippets/package_metadata_fields.html :language: jinja :end-before: {% block package_metadata_fields %} @@ -171,19 +171,19 @@ directory called ``package/snippets/package_basic_fields.html`` containing :language: jinja This adds our custom_text field to the editing form. Finally we want to display -our custom_text field on the dataset page. Add another file called +our custom_text field on the dataset page. Add another file called ``package/snippets/additional_info.html`` containing .. literalinclude:: ../../ckanext/example_idatasetform/templates/package/snippets/additional_info.html :language: jinja -This template overrides the default extras rendering on the dataset page +This template overrides the default extras rendering on the dataset page and replaces it to just display our custom field. -You're done! Make sure you have your plugin installed and setup as in the -`extension/tutorial`. Then run a development server and you should now have -an additional field called "Custom Text" when displaying and adding/editing a +You're done! Make sure you have your plugin installed and setup as in the +`extension/tutorial`. Then run a development server and you should now have +an additional field called "Custom Text" when displaying and adding/editing a dataset. Cleaning up the code @@ -191,7 +191,7 @@ Cleaning up the code Before we continue further, we can clean up the :py:meth:`~ckan.plugins.interfaces.IDatasetForm.create_package_schema` -and :py:meth:`~ckan.plugins.interfaces.IDatasetForm.update_package_schema`. +and :py:meth:`~ckan.plugins.interfaces.IDatasetForm.update_package_schema`. There is a bit of duplication that we could remove. Replace the two functions with: @@ -279,9 +279,9 @@ Otherwise this is the same as the single-parameter form above. Validators that need to access or update multiple fields may be written as a callable taking four parameters. -This form of validator is passed the all the fields and +This form of validator is passed to all the fields and errors in a "flattened" form. Validator must fetch -values from ``flattened_data`` may replace values in +values from ``flattened_data`` and may replace values in ``flattened_data``. The return value from this function is ignored. ``key`` is the flattened key for the field to which this validator was @@ -317,14 +317,14 @@ above your plugin class. This code block is taken from the ``example_idatsetform plugin``. ``create_country_codes`` tries to fetch the vocabulary country_codes using -:func:`~ckan.logic.action.get.vocabulary_show`. If it is not found it will +:func:`~ckan.logic.action.get.vocabulary_show`. If it is not found it will create it and iterate over the list of countries 'uk', 'ie', 'de', 'fr', 'es'. -For each of these a vocabulary tag is created using +For each of these a vocabulary tag is created using :func:`~ckan.logic.action.create.tag_create`, belonging to the vocabulary -``country_code``. +``country_code``. Although we have only defined five tags here, additional tags can be created -at any point by a sysadmin user by calling +at any point by a sysadmin user by calling :func:`~ckan.logic.action.create.tag_create` using the API or action functions. Add a second function below ``create_country_codes`` @@ -332,10 +332,10 @@ Add a second function below ``create_country_codes`` :pyobject: country_codes country_codes will call ``create_country_codes`` so that the ``country_codes`` -vocabulary is created if it does not exist. Then it calls -:func:`~ckan.logic.action.get.tag_list` to return all of our vocabulary tags -together. Now we have a way of retrieving our tag vocabularies and creating -them if they do not exist. We just need our plugin to call this code. +vocabulary is created if it does not exist. Then it calls +:func:`~ckan.logic.action.get.tag_list` to return all of our vocabulary tags +together. Now we have a way of retrieving our tag vocabularies and creating +them if they do not exist. We just need our plugin to call this code. Adding tags to the schema ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -352,7 +352,7 @@ convert the field in to our tag in a similar way to how we converted our field to extras earlier. In :py:meth:`~ckan.plugins.interfaces.IDatasetForm.show_package_schema` we convert from the tag back again but we have an additional line with another converter -containing +containing :py:func:`~ckan.logic.converters.free_tags_only`. We include this line so that vocab tags are not shown mixed with normal free tags. @@ -386,22 +386,22 @@ Adding custom fields to resources In order to customize the fields in a resource the schema for resources needs to be modified in a similar way to the datasets. The resource schema is nested in the dataset dict as package['resources']. We modify this dict in -a similar way to the dataset schema. Change ``_modify_package_schema`` to the +a similar way to the dataset schema. Change ``_modify_package_schema`` to the following. .. literalinclude:: ../../ckanext/example_idatasetform/plugin.py :pyobject: ExampleIDatasetFormPlugin._modify_package_schema :emphasize-lines: 14-16 -Update :py:meth:`~ckan.plugins.interfaces.IDatasetForm.show_package_schema` +Update :py:meth:`~ckan.plugins.interfaces.IDatasetForm.show_package_schema` similarly .. literalinclude:: ../../ckanext/example_idatasetform/plugin.py :pyobject: ExampleIDatasetFormPlugin.show_package_schema :emphasize-lines: 20-23 - -Save and reload your development server CKAN will take any additional keys from -the resource schema and save them the its extras field. The templates will + +Save and reload your development server CKAN will take any additional keys from +the resource schema and save them the its extras field. The templates will automatically check this field and display them in the resource_read page. Sorting by custom fields on the dataset search page @@ -414,8 +414,8 @@ search page to sort datasets by our custom field. Add a new file called :language: jinja :emphasize-lines: 16-17 -This overrides the search ordering drop down code block, the code is the -same as the default dataset search block but we are adding two additional lines +This overrides the search ordering drop down code block, the code is the +same as the default dataset search block but we are adding two additional lines that define the display name of that search ordering (e.g. Custom Field Ascending) and the SOLR sort ordering (e.g. custom_text asc). If you reload your development server you should be able to see these two additional sorting options From 999d4aa77505863013c7bd450695f79307aac34d Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Wed, 21 Oct 2015 17:36:48 +0100 Subject: [PATCH 303/307] Revert "[#2697] Make sure package_autocomplete uses LIKE with wildcards" --- ckan/logic/action/get.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index e6c2f30f33f..242edfbf6c9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1608,7 +1608,7 @@ def package_autocomplete(context, data_dict): limit = data_dict.get('limit', 10) q = data_dict['q'] - like_q = u"%%%s%%" % q + like_q = u"%s%%" % q query = model.Session.query(model.Package) query = query.filter(model.Package.state == 'active') From 43c18b83818fbdbac938990205e8eab62d8b56dd Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Oct 2015 10:26:32 +0100 Subject: [PATCH 304/307] Add CircleCI conf file --- circle.yml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 circle.yml diff --git a/circle.yml b/circle.yml new file mode 100644 index 00000000000..1c51d165445 --- /dev/null +++ b/circle.yml @@ -0,0 +1,39 @@ +machine: + + environment: + PIP_USE_MIRRORS: true + +dependencies: + override: + - pip install -r requirements.txt --allow-all-external + - pip install -r dev-requirements.txt --allow-all-external + - python setup.py develop + + post: + - npm install -g mocha-phantomjs@3.5.0 phantomjs@~1.9.1 + +database: + post: + - sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';" + - sudo -u postgres psql -c "CREATE USER datastore_default WITH PASSWORD 'pass';" + - sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;' + - sudo -u postgres psql -c 'CREATE DATABASE datastore_test WITH OWNER ckan_default;' + - sed -i -e 's/.*datastore.read_url.*/ckan.datastore.read_url = postgresql:\/\/datastore_default:pass@\/datastore_test/' test-core.ini + - paster datastore -c test-core.ini set-permissions | sudo -u postgres psql + + - cp -R /opt/solr-4.3.1 $HOME/solr + - cp ckan/config/solr/schema.xml $HOME/solr/example/solr/collection1/conf + - cd $HOME/solr/example; java -jar start.jar >> $HOME/solr.log: + background: true + + - paster db init -c test-core.ini + +test: + override: + - nosetests --ckan --reset-db --with-pylons=test-core.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext ckan ckanext + + post: + - paster serve test-core.ini: + background: true + - sleep 5 + - mocha-phantomjs http://localhost:5000/base/test/index.html From 80ddf337053308e8a38ab3c2b4adf5a82e4508ea Mon Sep 17 00:00:00 2001 From: LondonAppDev Date: Thu, 22 Oct 2015 10:54:35 +0100 Subject: [PATCH 305/307] Update adding-custom-fields.rst --- doc/extensions/adding-custom-fields.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/extensions/adding-custom-fields.rst b/doc/extensions/adding-custom-fields.rst index f66f2f46372..ab340c354cf 100644 --- a/doc/extensions/adding-custom-fields.rst +++ b/doc/extensions/adding-custom-fields.rst @@ -279,10 +279,10 @@ Otherwise this is the same as the single-parameter form above. Validators that need to access or update multiple fields may be written as a callable taking four parameters. -This form of validator is passed to all the fields and -errors in a "flattened" form. Validator must fetch -values from ``flattened_data`` and may replace values in -``flattened_data``. The return value from this function is ignored. +All fields and errors in a ``flattened`` form are passed to the +validator. The validator must fetch values from ``flattened_data`` +and may replace values in ``flattened_data``. The return value +from this function is ignored. ``key`` is the flattened key for the field to which this validator was applied. For example ``('notes',)`` for the dataset notes field or From 75a9120b10ca9643a1fe2d1b1aa64204cb052f84 Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 26 Oct 2015 10:29:54 +0000 Subject: [PATCH 306/307] Fix typo. --- ckan/lib/dictization/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/lib/dictization/__init__.py b/ckan/lib/dictization/__init__.py index 5cd97d9a2e6..61d03708461 100644 --- a/ckan/lib/dictization/__init__.py +++ b/ckan/lib/dictization/__init__.py @@ -115,7 +115,7 @@ def table_dict_save(table_dict, ModelClass, context): obj = None - unique_constriants = get_unique_constraints(table, context) + unique_constraints = get_unique_constraints(table, context) id = table_dict.get("id") @@ -123,8 +123,8 @@ def table_dict_save(table_dict, ModelClass, context): obj = session.query(ModelClass).get(id) if not obj: - unique_constriants = get_unique_constraints(table, context) - for constraint in unique_constriants: + unique_constraints = get_unique_constraints(table, context) + for constraint in unique_constraints: params = dict((key, table_dict.get(key)) for key in constraint) obj = session.query(ModelClass).filter_by(**params).first() if obj: From 8c977e3e8f30fdc06c159caaefe7fc98f4c1d168 Mon Sep 17 00:00:00 2001 From: Alex Palcuie Date: Mon, 26 Oct 2015 18:27:49 +0200 Subject: [PATCH 307/307] Use old style Travis builds --- ckan/pastertemplates/template/+dot+travis.yml_tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/pastertemplates/template/+dot+travis.yml_tmpl b/ckan/pastertemplates/template/+dot+travis.yml_tmpl index 1cc24d6ebc2..536ae829969 100644 --- a/ckan/pastertemplates/template/+dot+travis.yml_tmpl +++ b/ckan/pastertemplates/template/+dot+travis.yml_tmpl @@ -1,4 +1,5 @@ language: python +sudo: required python: - "2.6" - "2.7"