From 9370ceb3097db7183ed1e156c62f143b6ec1c359 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 14 Apr 2016 11:43:16 +0100 Subject: [PATCH 001/210] [#2955] Fix group feeds returning no datasets --- ckan/controllers/feed.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ckan/controllers/feed.py b/ckan/controllers/feed.py index 8720e5d2535..236b4c4c507 100644 --- a/ckan/controllers/feed.py +++ b/ckan/controllers/feed.py @@ -169,12 +169,17 @@ def _alternate_url(self, params, **kwargs): def _group_or_organization(self, obj_dict, is_org): data_dict, params = self._parse_url_params() - key = 'owner_org' if is_org else 'groups' - data_dict['fq'] = '%s:"%s"' % (key, obj_dict['id'],) - group_type = 'organization' - if not is_org: + if is_org: + key = 'owner_org' + value = obj_dict['id'] + group_type = 'organization' + else: + key = 'groups' + value = obj_dict['name'] group_type = 'group' + data_dict['fq'] = '{0}:"{1}"'.format(key, value) + item_count, results = _package_search(data_dict) navigation_urls = self._navigation_urls(params, From 5722d3d6bcf59585106baf42e5fd31c3962c72fc Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 14 Apr 2016 11:43:40 +0100 Subject: [PATCH 002/210] [#2955] Add tests for feed controller --- ckan/tests/controllers/test_feed.py | 51 ++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index 2293ae5118a..fb48a9548bb 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -1,12 +1,15 @@ from routes import url_for -from ckan import model import ckan.tests.helpers as helpers import ckan.tests.factories as factories class TestFeedNew(helpers.FunctionalTestBase): + @classmethod + def teardown_class(cls): + helpers.reset_db() + def test_atom_feed_page_zero_gives_error(self): group = factories.Group() offset = url_for(controller='feed', action='group', @@ -30,3 +33,49 @@ def test_atom_feed_page_not_int_gives_error(self): app = self._get_test_app() res = app.get(offset, status=400) assert '"page" parameter must be a positive integer' in res, res + + def test_general_atom_feed_works(self): + dataset = factories.Dataset() + offset = url_for(controller='feed', action='general') + app = self._get_test_app() + res = app.get(offset) + + assert '{0}'.format(dataset['title']) in res.body + + def test_group_atom_feed_works(self): + group = factories.Group() + dataset = factories.Dataset(groups=[{'id': group['id']}]) + offset = url_for(controller='feed', action='group', + id=group['name']) + app = self._get_test_app() + res = app.get(offset) + + assert '{0}'.format(dataset['title']) in res.body + + def test_organization_atom_feed_works(self): + group = factories.Organization() + dataset = factories.Dataset(owner_org=group['id']) + offset = url_for(controller='feed', action='organization', + id=group['name']) + app = self._get_test_app() + res = app.get(offset) + + assert '{0}'.format(dataset['title']) in res.body + + def test_custom_atom_feed_works(self): + dataset1 = factories.Dataset( + title='Test weekly', + extras=[{'key': 'frequency', 'value': 'weekly'}]) + dataset2 = factories.Dataset( + title='Test daily', + extras=[{'key': 'frequency', 'value': 'daily'}]) + offset = url_for(controller='feed', action='custom') + params = { + 'q': 'frequency:weekly' + } + app = self._get_test_app() + res = app.get(offset, params=params) + + assert '{0}'.format(dataset1['title']) in res.body + + assert '{0}'.format(dataset2['title']) not in res.body From 44a6783ca2b47e6f6a28345ed097b0877f224231 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 May 2016 11:10:37 +0100 Subject: [PATCH 003/210] [#3002] Only allow JSONP callback on the action API for GET requests Updated the tests from the legacy ones. --- ckan/controllers/api.py | 6 ++-- ckan/tests/controllers/test_api.py | 50 ++++++++++++++++++++++++++ ckan/tests/legacy/logic/test_action.py | 16 --------- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index de734483b86..a32ba722ef3 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -89,9 +89,9 @@ def _finish(self, status_int, response_data=None, else: response_msg = response_data # Support "JSONP" callback. - if status_int == 200 and 'callback' in request.params and \ - (request.method == 'GET' or - c.logic_function and request.method == 'POST'): + if (status_int == 200 and + 'callback' in request.params and + request.method == 'GET'): # escape callback to remove '<', '&', '>' chars callback = cgi.escape(request.params['callback']) response_msg = self._wrap_jsonp(callback, response_msg) diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 36b666d21aa..90aaa3dfe34 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -3,6 +3,7 @@ controller itself. ''' import json +import re from routes import url_for from nose.tools import assert_equal @@ -13,6 +14,9 @@ from ckan import model +eq_ = assert_equal + + class TestApiController(helpers.FunctionalTestBase): def test_unicode_in_error_message_works_ok(self): @@ -201,6 +205,52 @@ def test_config_option_list_access_sysadmin_jsonp(self): status=403, ) + def test_jsonp_works_on_get_requests(self): + + dataset1 = factories.Dataset() + dataset2 = factories.Dataset() + + url = url_for( + controller='api', + action='action', + logic_function='package_list', + ver='/3') + app = self._get_test_app() + res = app.get( + url=url, + params={'callback': 'my_callback'}, + ) + assert re.match('my_callback\(.*\);', res.body), res + # Unwrap JSONP callback (we want to look at the data). + msg = res.body[len('my_callback')+1:-2] + res_dict = json.loads(msg) + eq_(res_dict['success'], True) + eq_(sorted(res_dict['result']), + sorted([dataset1['name'], dataset2['name']])) + + def test_jsonp_does_not_work_on_post_requests(self): + + dataset1 = factories.Dataset() + dataset2 = factories.Dataset() + + url = url_for( + controller='api', + action='action', + logic_function='package_list', + ver='/3', + callback='my_callback', + ) + app = self._get_test_app() + res = app.post( + url=url, + ) + # The callback param is ignored and the normal response is returned + assert not res.body.startswith('my_callback') + res_dict = json.loads(res.body) + eq_(res_dict['success'], True) + eq_(sorted(res_dict['result']), + sorted([dataset1['name'], dataset2['name']])) + class TestRevisionSearch(helpers.FunctionalTestBase): diff --git a/ckan/tests/legacy/logic/test_action.py b/ckan/tests/legacy/logic/test_action.py index 2c109002bea..334a6b87c52 100644 --- a/ckan/tests/legacy/logic/test_action.py +++ b/ckan/tests/legacy/logic/test_action.py @@ -114,22 +114,6 @@ def test_01_package_list_private(self): assert 'public_dataset' in res assert not 'private_dataset' in res - def test_01_package_show_with_jsonp(self): - anna_id = model.Package.by_name(u'annakarenina').id - postparams = '%s=1' % json.dumps({'id': anna_id}) - res = self.app.post('/api/action/package_show?callback=jsoncallback', params=postparams) - - assert re.match('jsoncallback\(.*\);', res.body), res - # Unwrap JSONP callback (we want to look at the data). - msg = res.body[len('jsoncallback')+1:-2] - res_dict = json.loads(msg) - assert_equal(res_dict['success'], True) - assert "/api/3/action/help_show?name=package_show" in res_dict['help'] - pkg = res_dict['result'] - assert_equal(pkg['name'], 'annakarenina') - missing_keys = set(('title', 'groups')) - set(pkg.keys()) - assert not missing_keys, missing_keys - def test_02_package_autocomplete_match_name(self): postparams = '%s=1' % json.dumps({'q':'war', 'limit': 5}) res = self.app.post('/api/action/package_autocomplete', params=postparams) From 4ee76da6b05c9b34413d1f232369ac92d94bad49 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 May 2016 11:13:41 +0100 Subject: [PATCH 004/210] [#3002] Mention in docs that only GET is supported --- doc/api/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/index.rst b/doc/api/index.rst index 651a1dbeabd..fac6ede30ec 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -314,7 +314,7 @@ call. The function is named in the 'callback' parameter. For example: http://demo.ckan.org/api/3/action/package_show?id=adur_district_spending&callback=myfunction -.. todo :: This doesn't work with all functions. +.. note :: This only works for GET requests .. _api-examples: From 54aff38c9b99085fd2588e299e8b15b42aba5380 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 May 2016 14:09:56 +0100 Subject: [PATCH 005/210] [#3002] Fix PEP8 --- ckan/tests/controllers/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 90aaa3dfe34..38920acc724 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -222,7 +222,7 @@ def test_jsonp_works_on_get_requests(self): ) assert re.match('my_callback\(.*\);', res.body), res # Unwrap JSONP callback (we want to look at the data). - msg = res.body[len('my_callback')+1:-2] + msg = res.body[len('my_callback') + 1:-2] res_dict = json.loads(msg) eq_(res_dict['success'], True) eq_(sorted(res_dict['result']), From 02668cb8b24c29c94e12cc2629631fbc07312e5e Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 10 May 2016 10:33:13 +0100 Subject: [PATCH 006/210] [#3002] Smarter nose import cc @TkTech --- ckan/tests/controllers/test_api.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 38920acc724..258337a487a 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -6,7 +6,7 @@ import re from routes import url_for -from nose.tools import assert_equal +from nose.tools import assert_equal, eq_ import ckan.tests.helpers as helpers from ckan.tests.helpers import assert_in @@ -14,9 +14,6 @@ from ckan import model -eq_ = assert_equal - - class TestApiController(helpers.FunctionalTestBase): def test_unicode_in_error_message_works_ok(self): From b670bbfe0dc05b831ef8a4798f44784a4345ac7d Mon Sep 17 00:00:00 2001 From: Carl Lange Date: Mon, 16 May 2016 10:11:43 +0100 Subject: [PATCH 007/210] Add a submit_all command to datapusher. this allows you to add every resource of every package to the datastore. this is useful if you're setting up datastore for a a ckan that's already got datasets. Also renames the existing submit_all function to resubmit_all, because it would not act on resources that had not already been submitted to datastore. --- ckanext/datapusher/cli.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/ckanext/datapusher/cli.py b/ckanext/datapusher/cli.py index 7db5bc81711..2328b051f14 100644 --- a/ckanext/datapusher/cli.py +++ b/ckanext/datapusher/cli.py @@ -14,6 +14,9 @@ class DatapusherCommand(cli.CkanCommand): ignoring if their files haven't changed. submit - Submits all resources from the package identified by pkgname (either the short name or ID). + submit_all - Submit every package to the datastore. + This is useful if you're setting up datastore + for a ckan that already has datasets. ''' summary = __doc__.split('\n')[0] @@ -24,7 +27,12 @@ def command(self): self._confirm_or_abort() self._load_config() - self._submit_all() + self._resubmit_all() + elif self.args and self.args[0] == 'submit_all': + self._confirm_or_abort() + + self._load_config() + self._submit_all_packages() elif self.args and self.args[0] == 'submit': self._confirm_or_abort() @@ -49,10 +57,18 @@ def _confirm_or_abort(self): print "Aborting..." sys.exit(0) - def _submit_all(self): + def _resubmit_all(self): resources_ids = datastore_db.get_all_resources_ids_in_datastore() self._submit(resource_ids) + def _submit_all_packages(self): + # submit every package + # for each package in the package list, submit each resource w/ _submit_package + import ckan.model as model + package_list = p.toolkit.get_action('package_list') + for p_id in package_list({'model': model, 'ignore_auth': True}, {}): + self._submit_package(p_id) + def _submit_package(self, pkg_id): import ckan.model as model From 30a3c8485b8e639744e28864f1754be1e6c20a3c Mon Sep 17 00:00:00 2001 From: Jared Smith Date: Wed, 1 Jun 2016 09:31:03 -0700 Subject: [PATCH 008/210] [#2604] - File Upload UX Note: These changes only apply to data upload/linking. I've changed the label from 'File' to 'Data' when there's no file/link. When uploading a file from a local machine to a resource, the label will change to 'File'. When adding a link to a resource, the label will change to 'URL'. When editing a resource, if the data is an upload, the data field will only display the filename, rather than the url. The label for an upload will also display as 'File'. The remove data icon now uses text 'Remove' instead of an icon. --- .../base/javascript/modules/image-upload.js | 84 ++++++++++++++++--- ckan/public/base/less/forms.less | 4 +- .../package/snippets/resource_form.html | 2 +- 3 files changed, 74 insertions(+), 16 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index 7fef17dfcc4..3a622dc051c 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -5,7 +5,7 @@ this.ckan.module('image-upload', function($, _) { return { /* options object can be extended using data-module-* attributes */ options: { - is_url: true, + is_url: false, is_upload: false, field_upload: 'image_upload', field_url: 'image_url', @@ -17,6 +17,8 @@ this.ckan.module('image-upload', function($, _) { url: _('Link'), remove: _('Remove'), upload_label: _('Image'), + label_for_url: _('URL'), + label_for_upload: _('File'), upload_tooltip: _('Upload a file on your computer'), url_tooltip: _('Link to a URL on the internet (you can also link to an API)') } @@ -38,20 +40,23 @@ this.ckan.module('image-upload', function($, _) { // firstly setup the fields var field_upload = 'input[name="' + options.field_upload + '"]'; - var field_url = 'input[name="' + options.field_url + '"]'; - var field_clear = 'input[name="' + options.field_clear + '"]'; - var field_name = 'input[name="' + options.field_name + '"]'; + var field_url = 'input[name="' + options.field_url + '"]'; + var field_clear = 'input[name="' + options.field_clear + '"]'; + var field_name = 'input[name="' + options.field_name + '"]'; - this.input = $(field_upload, this.el); - this.field_url = $(field_url, this.el).parents('.control-group'); - this.field_image = this.input.parents('.control-group'); - this.field_url_input = $('input', this.field_url); - this.field_name = this.el.parents('form').find(field_name); + this.input = $(field_upload, this.el); + this.field_url = $(field_url, this.el).parents('.control-group'); + this.field_image = this.input.parents('.control-group'); + this.field_url_input = $('input', this.field_url); + this.field_name = this.el.parents('form').find(field_name); + // this is the location for the upload/link data/image label + this.label_location = $('label[for="field-image-url"]'); + // determines if the resource is a data resource + this.is_data_resource = (this.options.field_url == 'url') && (this.options.field_upload == 'upload'); // Is there a clear checkbox on the form already? var checkbox = $(field_clear, this.el); if (checkbox.length > 0) { - options.is_upload = true; checkbox.parents('.control-group').remove(); } @@ -60,7 +65,7 @@ this.ckan.module('image-upload', function($, _) { .appendTo(this.el); // Button to set the field to be a URL - this.button_url = $(' '+this.i18n('url')+'') + this.button_url = $(''+this.i18n('url')+'') .prop('title', this.i18n('url_tooltip')) .on('click', this._onFromWeb) .insertAfter(this.input); @@ -70,12 +75,12 @@ this.ckan.module('image-upload', function($, _) { .insertAfter(this.input); // Button for resetting the form when there is a URL set - $('') + $(''+this.i18n('remove')+'') .prop('title', this.i18n('remove')) .on('click', this._onRemove) .insertBefore(this.field_url_input); - // Update the main label + // Update the main label (this is displayed when no data/image has been uploaded/linked) $('label[for="field-image-upload"]').text(options.upload_label || this.i18n('upload_label')); // Setup the file input @@ -105,25 +110,71 @@ this.ckan.module('image-upload', function($, _) { if (options.is_url) { this._showOnlyFieldUrl(); + + this._updateUrlLabel(this.i18n('label_for_url')); } else if (options.is_upload) { this._showOnlyFieldUrl(); + this.field_url_input.prop('readonly', true); + // If the data is an uploaded file, the filename will display rather than whole url of the site + var filename = this._fileNameFromUpload(this.field_url_input.val()); + this.field_url_input.val(filename); + + this._updateUrlLabel(this.i18n('label_for_upload')); } else { this._showOnlyButtons(); } }, + /* Quick way of getting just the filename from the uri of the resource data + * + * url - The url of the uploaded data file + * + * Returns String. + */ + _fileNameFromUpload: function(url) { + // remove fragment (#) + url = url.substring(0, (url.indexOf("#") === -1) ? url.length : url.indexOf("#")); + // remove query string + url = url.substring(0, (url.indexOf("?") === -1) ? url.length : url.indexOf("?")); + // extract the filename + url = url.substring(url.lastIndexOf("/") + 1, url.length); + + return url; // filename + }, + + /* Update the `this.label_location` text + * + * If the upload/link is for a data resource, rather than an image, + * the text for label[for="field-image-url"] will be updated. + * + * label_text - The text for the label of an uploaded/linked resource + * + * Returns nothing. + */ + _updateUrlLabel: function(label_text) { + if (! this.is_data_resource) { + return; + } + + this.label_location.text(label_text); + }, + /* Event listener for when someone sets the field to URL mode * * Returns nothing. */ _onFromWeb: function() { this._showOnlyFieldUrl(); + this.field_url_input.focus() .on('blur', this._onFromWebBlur); + if (this.options.is_upload) { this.field_clear.val('true'); } + + this._updateUrlLabel(this.i18n('label_for_url')); }, /* Event listener for resetting the field back to the blank state @@ -132,8 +183,10 @@ this.ckan.module('image-upload', function($, _) { */ _onRemove: function() { this._showOnlyButtons(); + this.field_url_input.val(''); this.field_url_input.prop('readonly', false); + this.field_clear.val('true'); }, @@ -145,9 +198,14 @@ this.ckan.module('image-upload', function($, _) { var file_name = this.input.val().split(/^C:\\fakepath\\/).pop(); this.field_url_input.val(file_name); this.field_url_input.prop('readonly', true); + this.field_clear.val(''); + this._showOnlyFieldUrl(); + this._autoName(file_name); + + this._updateUrlLabel(this.i18n('label_for_upload')); }, /* Show only the buttons, hiding all others diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 66d0901dc69..2800dde3e98 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -736,7 +736,7 @@ textarea { .js .image-upload { #field-image-url { - padding-right: 29px; + padding-right: 90px; } #field-image-upload { cursor: pointer; @@ -763,7 +763,7 @@ textarea { margin-right: 0; top: 4px; right: 5px; - padding: 0 4px; + padding: 0 12px; .border-radius(100px); .icon-remove { margin-right: 0; diff --git a/ckan/templates/package/snippets/resource_form.html b/ckan/templates/package/snippets/resource_form.html index 2a737064c02..29108012b68 100644 --- a/ckan/templates/package/snippets/resource_form.html +++ b/ckan/templates/package/snippets/resource_form.html @@ -21,7 +21,7 @@ {% set is_upload = (data.url_type == 'upload') %} {{ form.image_upload(data, errors, field_url='url', field_upload='upload', field_clear='clear_upload', is_upload_enabled=h.uploads_enabled(), is_url=data.url and not is_upload, is_upload=is_upload, - upload_label=_('File'), url_label=_('URL')) }} + upload_label=_('Data'), url_label=_('URL'), placeholder=_('http://example.com/external-data.csv')) }} {% endblock %} {% block basic_fields_name %} From 67129e15953f0f25c8e778faabe257d9075972b2 Mon Sep 17 00:00:00 2001 From: Jared Smith Date: Thu, 12 May 2016 15:04:27 -0700 Subject: [PATCH 009/210] [#2604] - File Upload UX CSS Updated main.css --- ckan/public/base/css/fuchsia.css | 4 ++-- ckan/public/base/css/green.css | 4 ++-- ckan/public/base/css/main.css | 4 ++-- ckan/public/base/css/maroon.css | 4 ++-- ckan/public/base/css/red.css | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ckan/public/base/css/fuchsia.css b/ckan/public/base/css/fuchsia.css index 536788526d4..2308c828e38 100644 --- a/ckan/public/base/css/fuchsia.css +++ b/ckan/public/base/css/fuchsia.css @@ -6891,7 +6891,7 @@ textarea { box-shadow: none; } .js .image-upload #field-image-url { - padding-right: 29px; + padding-right: 90px; } .js .image-upload #field-image-upload { cursor: pointer; @@ -6922,7 +6922,7 @@ textarea { margin-right: 0; top: 4px; right: 5px; - padding: 0 4px; + padding: 0 12px; -webkit-border-radius: 100px; -moz-border-radius: 100px; border-radius: 100px; diff --git a/ckan/public/base/css/green.css b/ckan/public/base/css/green.css index 3b3143d2c02..78dddffbaeb 100644 --- a/ckan/public/base/css/green.css +++ b/ckan/public/base/css/green.css @@ -6891,7 +6891,7 @@ textarea { box-shadow: none; } .js .image-upload #field-image-url { - padding-right: 29px; + padding-right: 90px; } .js .image-upload #field-image-upload { cursor: pointer; @@ -6922,7 +6922,7 @@ textarea { margin-right: 0; top: 4px; right: 5px; - padding: 0 4px; + padding: 0 12px; -webkit-border-radius: 100px; -moz-border-radius: 100px; border-radius: 100px; diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 97b9f35cfbf..b33393cc971 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -6876,7 +6876,7 @@ textarea { box-shadow: none; } .js .image-upload #field-image-url { - padding-right: 29px; + padding-right: 90px; } .js .image-upload #field-image-upload { cursor: pointer; @@ -6907,7 +6907,7 @@ textarea { margin-right: 0; top: 4px; right: 5px; - padding: 0 4px; + padding: 0 12px; -webkit-border-radius: 100px; -moz-border-radius: 100px; border-radius: 100px; diff --git a/ckan/public/base/css/maroon.css b/ckan/public/base/css/maroon.css index dbeb455f691..39aa8d02444 100644 --- a/ckan/public/base/css/maroon.css +++ b/ckan/public/base/css/maroon.css @@ -6891,7 +6891,7 @@ textarea { box-shadow: none; } .js .image-upload #field-image-url { - padding-right: 29px; + padding-right: 90px; } .js .image-upload #field-image-upload { cursor: pointer; @@ -6922,7 +6922,7 @@ textarea { margin-right: 0; top: 4px; right: 5px; - padding: 0 4px; + padding: 0 12px; -webkit-border-radius: 100px; -moz-border-radius: 100px; border-radius: 100px; diff --git a/ckan/public/base/css/red.css b/ckan/public/base/css/red.css index cb43489b74a..9759c39445a 100644 --- a/ckan/public/base/css/red.css +++ b/ckan/public/base/css/red.css @@ -6891,7 +6891,7 @@ textarea { box-shadow: none; } .js .image-upload #field-image-url { - padding-right: 29px; + padding-right: 90px; } .js .image-upload #field-image-upload { cursor: pointer; @@ -6922,7 +6922,7 @@ textarea { margin-right: 0; top: 4px; right: 5px; - padding: 0 4px; + padding: 0 12px; -webkit-border-radius: 100px; -moz-border-radius: 100px; border-radius: 100px; From 7b6dbed7ba8e735fce2bad52beccede6cd1ef373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Thu, 2 Jun 2016 13:31:36 +0200 Subject: [PATCH 010/210] [#3027] use commit from 3027 and add test --- ckan/controllers/user.py | 2 +- ckan/tests/controllers/test_user.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index 110c27841be..f70eab11b2c 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -180,7 +180,7 @@ def new(self, data=None, errors=None, error_summary=None): if context['save'] and not data: return self._save_new(context) - if c.user and not data: + if c.user and not data and not authz.is_sysadmin(c.user): # #1799 Don't offer the registration form if already logged in return render('user/logout_first.html') diff --git a/ckan/tests/controllers/test_user.py b/ckan/tests/controllers/test_user.py index 5929dc528d6..624a23451d1 100644 --- a/ckan/tests/controllers/test_user.py +++ b/ckan/tests/controllers/test_user.py @@ -224,6 +224,16 @@ def test_user_edit_not_logged_in(self): status=403 ) + def test_create_user_as_sysadmin(self): + sysadmin = factories.Sysadmin() + app = self._get_test_app() + env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} + response = app.get( + url=url_for(controller='user', action='register'), + extra_environ=env, + ) + assert "user-register-form" in response.forms + def test_edit_user(self): user = factories.User(password='pass') app = self._get_test_app() From a75630da9818a54e84fe1fa6574bab2e3572404d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Thu, 2 Jun 2016 14:32:14 +0200 Subject: [PATCH 011/210] [#3027] redirect to user feed after creation --- ckan/controllers/user.py | 12 +++++++-- ckan/tests/controllers/test_user.py | 41 ++++++++++++++++++++++------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index f70eab11b2c..c7637fa122d 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -166,7 +166,8 @@ def new(self, data=None, errors=None, error_summary=None): '''GET to display a form for registering a new user. or POST the form data to actually do the user registration. ''' - context = {'model': model, 'session': model.Session, + context = {'model': model, + 'session': model.Session, 'user': c.user, 'auth_user_obj': c.userobj, 'schema': self._new_form_to_db_schema(), @@ -264,7 +265,14 @@ def _save_new(self, context): h.flash_success(_('User "%s" is now registered but you are still ' 'logged in as "%s" from before') % (data_dict['name'], c.user)) - return render('user/logout_first.html') + if authz.is_sysadmin(c.user): + # the sysadmin created a new user. We redirect him to the + # activity page for the newly created user + h.redirect_to(controller='user', + action='activity', + id=data_dict['name']) + else: + return render('user/logout_first.html') def edit(self, id=None, data=None, errors=None, error_summary=None): context = {'save': 'save' in request.params, diff --git a/ckan/tests/controllers/test_user.py b/ckan/tests/controllers/test_user.py index 624a23451d1..cedeca245ce 100644 --- a/ckan/tests/controllers/test_user.py +++ b/ckan/tests/controllers/test_user.py @@ -59,6 +59,37 @@ def test_register_user_bad_password(self): response = form.submit('save') assert_true('The passwords you entered do not match' in response) + def test_create_user_as_sysadmin(self): + admin_pass = 'pass' + sysadmin = factories.Sysadmin(password=admin_pass) + app = self._get_test_app() + + # Have to do an actual login as this test relies on repoze + # cookie handling. + + # get the form + response = app.get('/user/login') + # ...it's the second one + login_form = response.forms[1] + # fill it in + login_form['login'] = sysadmin['name'] + login_form['password'] = admin_pass + # submit it + login_form.submit('save') + + response = app.get( + url=url_for(controller='user', action='register'), + ) + assert "user-register-form" in response.forms + form = response.forms['user-register-form'] + form['name'] = 'newestuser' + form['fullname'] = 'Newest User' + form['email'] = 'test@test.com' + form['password1'] = 'testpassword' + form['password2'] = 'testpassword' + response2 = form.submit('save') + assert '/user/activity' in response2.location + class TestLoginView(helpers.FunctionalTestBase): def test_registered_user_login(self): @@ -224,16 +255,6 @@ def test_user_edit_not_logged_in(self): status=403 ) - def test_create_user_as_sysadmin(self): - sysadmin = factories.Sysadmin() - app = self._get_test_app() - env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} - response = app.get( - url=url_for(controller='user', action='register'), - extra_environ=env, - ) - assert "user-register-form" in response.forms - def test_edit_user(self): user = factories.User(password='pass') app = self._get_test_app() From a0a8f68732e9a194ad8354ede643060162b4f7cf Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Mon, 23 May 2016 15:35:01 +0200 Subject: [PATCH 012/210] Document CKAN Unicode handling. --- doc/contributing/index.rst | 1 + doc/contributing/python.rst | 13 +++ doc/contributing/unicode.rst | 160 +++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 doc/contributing/unicode.rst diff --git a/doc/contributing/index.rst b/doc/contributing/index.rst index 038c865596c..3a40d6b0b85 100644 --- a/doc/contributing/index.rst +++ b/doc/contributing/index.rst @@ -34,6 +34,7 @@ of contributions to CKAN: javascript python string-i18n + unicode testing frontend/index diff --git a/doc/contributing/python.rst b/doc/contributing/python.rst index e5d6ae015da..b557dae81ff 100644 --- a/doc/contributing/python.rst +++ b/doc/contributing/python.rst @@ -108,6 +108,19 @@ replacement field, for example:: .. _new .format() method: http://docs.python.org/2/library/stdtypes.html#str.format + +Unicode handling +---------------- +CKAN strives to only use Unicode internally (via the ``unicode`` type) and to +convert to/from ASCII at the interface to other systems and libraries if +necessary. + +.. seealso:: + + :doc:`unicode` + Details on Unicode handling in CKAN + + .. _docstrings: Docstrings diff --git a/doc/contributing/unicode.rst b/doc/contributing/unicode.rst new file mode 100644 index 00000000000..eb4a803d72b --- /dev/null +++ b/doc/contributing/unicode.rst @@ -0,0 +1,160 @@ +================ +Unicode handling +================ +This document explains how Unicode and related issues are handled in CKAN. +For a general introduction to Unicode and Unicode handling in Python 2 please +read the `Python 2 Unicode HOWTO`_. Since Unicode handling differs greatly +between Python 2 and Python 3 you might also be interested in the +`Python 3 Unicode HOWTO`_. + +.. _Python 2 Unicode HOWTO: https://docs.python.org/2/howto/unicode.html +.. _Python 3 Unicode HOWTO: https://docs.python.org/3/howto/unicode.html + +.. note:: + + This document describes the intended future state of Unicode handling in + CKAN. For historic reasons, some existing code does not yet follow the + rules described here. + + *New code should always comply with the rules in this document. Exceptions + must be documented.* + + +Overall Strategy +---------------- +CKAN only uses Unicode internally (``unicode`` on Python 2). Conversion to/from +ASCII strings happens on the boundary to other systems/libaries if necessary. + + +Encoding of Python files +------------------------ +Files containing Python source code (``*.py``) must be encoded using UTF-8, and +the encoding must be declared using the following header:: + + # encoding: utf-8 + +This line must be the first or second line in the file. See `PEP 263`_ for +details. + +.. _PEP 263: https://www.python.org/dev/peps/pep-0263/ + + +String literals +--------------- +String literals are string values given directly in the source code (as opposed +to strings variables read from a file, received via argument, etc.). In +Python 2, string literals by default have type ``str``. They can be changed to +``unicode`` by adding a ``u`` prefix. In addition, the ``b`` prefix can be used +to explicitly mark a literal as ``str``:: + + x = "I'm a str literal" + y = u"I'm a unicode literal" + z = b"I'm also a str literal" + +In CKAN, every string literal must carry either a ``u`` or a ``b`` prefix. +While the latter is redundant in Python 2, it makes the developer's intention +explicit and eases a future migration to Python 3. + +This rule also holds for *raw strings*, which are created using an ``r`` +prefix. Simply use ``ur`` instead:: + + m = re.match(ur'A\s+Unicode\s+pattern') + +For more information on string prefixes please refer to the +`Python documentation`_. + +.. _Python documentation: https://docs.python.org/2.7/reference/lexical_analysis.html#string-literals + +.. note:: + + The ``unicode_literals`` `future statement`_ is *not* used in CKAN. + +.. _future statement: https://docs.python.org/2/reference/simple_stmts.html#future + + +Best Practices +-------------- + +Use ``io.open`` to open text files +``````````````````````````````````` +When opening text (not binary) files you should use `io.open`_ instead of +``open``. This allows you to specify the file's encoding and reads will return +``unicode`` instead of ``str``:: + + import io + + with io.open(b'my_file.txt', u'r', encoding=u'utf-8') as f: + text = f.read() # contents is automatically decoded + # to unicode using UTF-8 + +.. _io.open: https://docs.python.org/2/library/io.html#io.open + +Text files should be encoded using UTF-8 if possible. + + +Normalize strings before comparing them +``````````````````````````````````````` +For many characters, Unicode offers multiple descriptions. For example, a small +latin ``e`` with an acute accent (``é``) can either be specified using its +dedicated code point (`U+00E9`_) or by combining the code points for ``e`` +(`U+0065`_) and the accent (`U+0301`_). Both variants will look the same but +are different from a numerical point of view:: + + >>> x = u'\N{LATIN SMALL LETTER E WITH ACUTE}' + >>> y = u'\N{LATIN SMALL LETTER E}\N{COMBINING ACUTE ACCENT}' + >>> print x, y + é é + >>> print repr(x), repr(y) + u'\xe9' u'e\u0301' + >>> x == y + False + +.. _U+00E9: http://www.fileformat.info/info/unicode/char/e9 +.. _U+0065: http://www.fileformat.info/info/unicode/char/0065 +.. _U+0301: http://www.fileformat.info/info/unicode/char/0301 + +Therefore, if you want to compare two Unicode strings based on their characters +you need to normalize them first using `unicodedata.normalize`_:: + + >>> from unicodedata import normalize + >>> x_norm = normalize(u'NFC', x) + >>> y_norm = normalize(u'NFC', y) + >>> print x_norm, y_norm + é é + >>> print repr(x_norm), repr(y_norm) + u'\xe9' u'\xe9' + >>> x_norm == y_norm + True + +.. _unicodedata.normalize: https://docs.python.org/2/library/unicodedata.html#unicodedata.normalize + + +Use the Unicode flag in regular expressions +``````````````````````````````````````````` +By default, the character classes of Python's `re`_ module (``\w``, ``\d``, +...) only match ASCII-characters. For example, ``\w`` (alphanumeric character) +does, by default, not match ``ö``:: + + >>> print re.match(ur'^\w$', u'ö') + None + +Therefore, you need to explicitly activate Unicode mode by passing the `re.U`_ +flag:: + + >>> print re.match(ur'^\w$', u'ö', re.U) + <_sre.SRE_Match object at 0xb60ea2f8> + +The type of the values returned by ``re.split``, ``re.MatchObject.group``, etc. +depends on the type of the input string:: + + >>> re.split(ur'\W+', b'Just a string!', flags=re.U) + ['Just', 'a', 'string', ''] + + >>> re.split(ur'\W+', u'Just some Unicode!', flags=re.U) + [u'Just', u'some', u'Unicode', u''] + +Note that the type of the *pattern string* does not influence the return type. + +.. _re: https://docs.python.org/2/library/re.html +.. _re.U: https://docs.python.org/2/library/re.html#re.U + From 7cf95acbc01c65d666dcf32a0f3c52642c1c4931 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Tue, 24 May 2016 17:07:03 +0200 Subject: [PATCH 013/210] [#3046] prompt user to create organization --- ckan/templates/package/new.html | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ckan/templates/package/new.html b/ckan/templates/package/new.html index a8eea92ca48..c040d61f928 100644 --- a/ckan/templates/package/new.html +++ b/ckan/templates/package/new.html @@ -1,3 +1,23 @@ {% extends "package/base_form_page.html" %} {% block subtitle %}{{ _('Create Dataset') }}{% endblock %} + + +{% if not h.organizations_available('create_dataset') %} + +{% block primary_content %} +
+ {% block page_header %}{% endblock %} +
+ {% block primary_content_inner %} +

{{ _('Before you can create a dataset you need to create an organization first.') }}

+ + + {{ _('Create a new organization') }} + + {% endblock %} +
+
+{% endblock %} + +{% endif %} \ No newline at end of file From 7227b7026cb47dab07912ce2ea837997dbb76d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Wed, 25 May 2016 09:27:57 +0200 Subject: [PATCH 014/210] [#3046] Fix typo in exception message --- ckan/logic/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/logic/validators.py b/ckan/logic/validators.py index c41bd23f5d1..ad8a91c5a8c 100644 --- a/ckan/logic/validators.py +++ b/ckan/logic/validators.py @@ -29,7 +29,7 @@ def owner_org_validator(key, data, errors, context): if value is missing or value is None: if not authz.check_config_permission('create_unowned_dataset'): - raise Invalid(_('A organization must be supplied')) + raise Invalid(_('An organization must be provided')) data.pop(key, None) raise df.StopOnError @@ -38,7 +38,7 @@ def owner_org_validator(key, data, errors, context): user = model.User.get(user) if value == '': if not authz.check_config_permission('create_unowned_dataset'): - raise Invalid(_('A organization must be supplied')) + raise Invalid(_('An organization must be provided')) return group = model.Group.get(value) From 48ffbf54a674b1bbeb1486012cb6b9e8a730a97d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Wed, 25 May 2016 13:35:15 +0200 Subject: [PATCH 015/210] [#3046] extract warning into own page --- ckan/templates/package/new.html | 25 +++++------------ .../snippets/cannot_create_package.html | 27 +++++++++++++++++++ 2 files changed, 33 insertions(+), 19 deletions(-) create mode 100644 ckan/templates/package/snippets/cannot_create_package.html diff --git a/ckan/templates/package/new.html b/ckan/templates/package/new.html index c040d61f928..9ef0635c727 100644 --- a/ckan/templates/package/new.html +++ b/ckan/templates/package/new.html @@ -1,23 +1,10 @@ -{% extends "package/base_form_page.html" %} +{% if not h.organizations_available('create_dataset') + and not h.check_config_permission('ckan.auth.create_unowned_dataset') %} -{% block subtitle %}{{ _('Create Dataset') }}{% endblock %} + {% include "package/snippets/cannot_create_package.html" %} +{% else %} + {% extends "package/base_form_page.html" %} -{% if not h.organizations_available('create_dataset') %} - -{% block primary_content %} -
- {% block page_header %}{% endblock %} -
- {% block primary_content_inner %} -

{{ _('Before you can create a dataset you need to create an organization first.') }}

- - - {{ _('Create a new organization') }} - - {% endblock %} -
-
-{% endblock %} - + {% block subtitle %}{{ _('Create Dataset') }}{% endblock %} {% endif %} \ No newline at end of file diff --git a/ckan/templates/package/snippets/cannot_create_package.html b/ckan/templates/package/snippets/cannot_create_package.html new file mode 100644 index 00000000000..b3711571bbf --- /dev/null +++ b/ckan/templates/package/snippets/cannot_create_package.html @@ -0,0 +1,27 @@ +{% extends "package/base_form_page.html" %} + +{% block primary_content %} + +{% block page_header %}{% endblock %} +
+
+ {% block primary_content_inner %} + {% if h.check_access('organization_create') %} +
{{ _('Before you can create a dataset you need to create an organization.') }}
+ + + {{ _('Create a new organization') }} + + + {% else %} +
+

{{ _('There are no organizations to which you can assign this dataset.') }}

+

{{ _('Ask a system administrator to create an organization before you can continue.') }}

+
+ + {% endif %} + + {% endblock %} +
+
+{% endblock %} \ No newline at end of file From ff93ffbfe0eb61b9c8b4dca7ea6125b4d6f46df6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Wed, 25 May 2016 15:35:29 +0200 Subject: [PATCH 016/210] [#3046] add tests --- ckan/tests/controllers/test_package.py | 38 ++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index cab8b9e4092..ff65b5537ef 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -8,6 +8,7 @@ assert_true, ) +from mock import patch from routes import url_for import ckan.model as model @@ -40,6 +41,43 @@ def test_form_renders(self): env, response = _get_package_new_page(app) assert_true('dataset-edit' in response.forms) + @helpers.change_config('ckan.auth.create_unowned_dataset', 'false') + def test_needs_organization_but_no_organizations_has_button(self,): + ''' Scenario: The settings say every dataset needs an organization + but there are no organizations. If the user is allowed to create an + organization they should be prompted to do so when they try to create + a new dataset''' + app = self._get_test_app() + sysadmin = factories.Sysadmin() + + env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} + response = app.get( + url=url_for(controller='package', action='new'), + extra_environ=env + ) + assert 'dataset-edit' not in response.forms + assert url_for(controller='organization', action='new') in response + + @patch('ckan.logic.auth.create.organization_create', + return_value={'success': False}) + @helpers.change_config('ckan.auth.create_unowned_dataset', 'false') + def test_needs_organization_but_no_organizations_no_button(self, *args): + ''' Scenario: The settings say every dataset needs an organization + but there are no organizations. If the user is not allowed to create an + organization they should be told to ask the admin but no link should be + presented''' + app = self._get_test_app() + sysadmin = factories.Sysadmin() + + env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} + response = app.get( + url=url_for(controller='package', action='new'), + extra_environ=env + ) + assert 'dataset-edit' not in response.forms + assert url_for(controller='organization', action='new') not in response + assert 'Ask a system administrator' in response + def test_name_required(self): app = self._get_test_app() env, response = _get_package_new_page(app) From 42632dc3aa0d42765b3cec0ad0cab821b70f5c15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Tue, 7 Jun 2016 13:49:18 +0200 Subject: [PATCH 017/210] [#3046] improve patching in test --- ckan/tests/controllers/test_package.py | 29 +++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index ff65b5537ef..81431f3e74d 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -8,7 +8,7 @@ assert_true, ) -from mock import patch +from mock import patch, MagicMock from routes import url_for import ckan.model as model @@ -58,22 +58,37 @@ def test_needs_organization_but_no_organizations_has_button(self,): assert 'dataset-edit' not in response.forms assert url_for(controller='organization', action='new') in response - @patch('ckan.logic.auth.create.organization_create', - return_value={'success': False}) @helpers.change_config('ckan.auth.create_unowned_dataset', 'false') - def test_needs_organization_but_no_organizations_no_button(self, *args): + @helpers.change_config('ckan.auth.user_create_organizations', 'false') + def test_needs_organization_but_no_organizations_no_button(self): ''' Scenario: The settings say every dataset needs an organization but there are no organizations. If the user is not allowed to create an organization they should be told to ask the admin but no link should be - presented''' + presented. Note: This cannot happen with the default ckan and requires + a plugin to overwrite the package_create behavior''' + + # Make sure that the patched `package_create` is actually looked at + # by the authentication functions. + from ckan.authz import clear_auth_functions_cache + p = patch('ckan.logic.auth.create.package_create', + new=MagicMock(return_value={'success': True})) + p.start() + clear_auth_functions_cache() + app = self._get_test_app() - sysadmin = factories.Sysadmin() + user = factories.User() - env = {'REMOTE_USER': sysadmin['name'].encode('ascii')} + env = {'REMOTE_USER': user['name'].encode('ascii')} response = app.get( url=url_for(controller='package', action='new'), extra_environ=env ) + + # Stop patching `package_create` and make sure that the other + # tests are not influenced by clearing the cache again + p.stop() + clear_auth_functions_cache() + assert 'dataset-edit' not in response.forms assert url_for(controller='organization', action='new') not in response assert 'Ask a system administrator' in response From 71ee7c1a5a427b34cf3878727fcd5208d78b69a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Tue, 7 Jun 2016 15:14:08 +0200 Subject: [PATCH 018/210] [#3046] create mock_auth helper function and use it --- ckan/tests/controllers/test_package.py | 20 +++-------- ckan/tests/helpers.py | 47 ++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index 81431f3e74d..aac6ba8b864 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -42,7 +42,7 @@ def test_form_renders(self): assert_true('dataset-edit' in response.forms) @helpers.change_config('ckan.auth.create_unowned_dataset', 'false') - def test_needs_organization_but_no_organizations_has_button(self,): + def test_needs_organization_but_no_organizations_has_button(self): ''' Scenario: The settings say every dataset needs an organization but there are no organizations. If the user is allowed to create an organization they should be prompted to do so when they try to create @@ -58,22 +58,17 @@ def test_needs_organization_but_no_organizations_has_button(self,): assert 'dataset-edit' not in response.forms assert url_for(controller='organization', action='new') in response + @helpers.mock_auth('ckan.logic.auth.create.package_create') @helpers.change_config('ckan.auth.create_unowned_dataset', 'false') @helpers.change_config('ckan.auth.user_create_organizations', 'false') - def test_needs_organization_but_no_organizations_no_button(self): + def test_needs_organization_but_no_organizations_no_button(self, + mock_p_create): ''' Scenario: The settings say every dataset needs an organization but there are no organizations. If the user is not allowed to create an organization they should be told to ask the admin but no link should be presented. Note: This cannot happen with the default ckan and requires a plugin to overwrite the package_create behavior''' - - # Make sure that the patched `package_create` is actually looked at - # by the authentication functions. - from ckan.authz import clear_auth_functions_cache - p = patch('ckan.logic.auth.create.package_create', - new=MagicMock(return_value={'success': True})) - p.start() - clear_auth_functions_cache() + mock_p_create.return_value = {'success': True} app = self._get_test_app() user = factories.User() @@ -84,11 +79,6 @@ def test_needs_organization_but_no_organizations_no_button(self): extra_environ=env ) - # Stop patching `package_create` and make sure that the other - # tests are not influenced by clearing the cache again - p.stop() - clear_auth_functions_cache() - assert 'dataset-edit' not in response.forms assert url_for(controller='organization', action='new') not in response assert 'Ask a system administrator' in response diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 62f8df16591..e43093076f2 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -328,6 +328,53 @@ def wrapper(*args, **kwargs): return decorator +def mock_auth(auth_function_path): + ''' + Decorator to easily mock a CKAN auth method in the context of a test + function + + It adds a mock object for the provided auth_function_path as a parameter to + the test function. + + Essentially it makes sure that `ckan.authz.clear_auth_functions_cache` is + called before and after to make sure that the auth functions pick up + the newly changed values. + + Usage:: + + @helpers.mock_auth('ckan.logic.auth.create.package_create') + def test_mock_package_create(self, mock_package_create): + from ckan import logic + mock_package_create.return_value = {'success': True} + + # package_create is mocked + eq_(logic.check_access('package_create', {}), True) + + assert mock_package_create.called + + :param action_name: the full path to the auth function to be mocked, + e.g. ``ckan.logic.auth.create.package_create`` + :type action_name: string + + ''' + from ckan.authz import clear_auth_functions_cache + + def decorator(func): + def wrapper(*args, **kwargs): + + try: + with mock.patch(auth_function_path) as mocked_auth: + clear_auth_functions_cache() + new_args = args + tuple([mocked_auth]) + return_value = func(*new_args, **kwargs) + finally: + clear_auth_functions_cache() + return return_value + + return nose.tools.make_decorator(func)(wrapper) + return decorator + + def mock_action(action_name): ''' Decorator to easily mock a CKAN action in the context of a test function From a2e30febe25aabcbcb34b4678489655086fa81f2 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 10 Jun 2016 11:25:47 +0100 Subject: [PATCH 019/210] [#3103] Use helper for version instead of context Instead of adding the ckan version as a value on `c`, this commit adds a new template helper that returns `ckan.__version__`. --- ckan/lib/base.py | 1 - ckan/lib/helpers.py | 9 ++++++++- ckan/templates/base.html | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 72446b366be..01aaa0b04dd 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -197,7 +197,6 @@ class BaseController(WSGIController): def __before__(self, action, **params): c.__timer = time.time() - c.__version__ = ckan.__version__ app_globals.app_globals._check_uptodate() self._identify_user() diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 5ae9929416a..9b71151f489 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -30,8 +30,8 @@ from routes import redirect_to as _redirect_to from routes import url_for as _routes_default_url_for import i18n -import ckan.exceptions +import ckan.exceptions import ckan.lib.fanstatic_resources as fanstatic_resources import ckan.model as model import ckan.lib.formatters as formatters @@ -41,6 +41,7 @@ import ckan.lib.uploader as uploader import ckan.authz as authz import ckan.plugins as p +import ckan from ckan.common import _, ungettext, g, c, request, session, json @@ -364,6 +365,12 @@ def lang(): return request.environ.get('CKAN_LANG') +@core_helper +def ckan_version(): + '''Return CKAN version''' + return ckan.__version__ + + @core_helper def lang_native_name(lang=None): ''' Return the langage name currently used in it's localised form diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 97f9301d71c..2d13f4eb520 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -26,7 +26,7 @@ #} {%- block meta -%} - {% block meta_generator %}{% endblock %} + {% block meta_generator %}{% endblock %} {% block meta_viewport %}{% endblock %} {%- endblock -%} From 62d1e67e309d47f96d3c42af5567a8fa4757a616 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sat, 11 Jun 2016 23:04:38 +0200 Subject: [PATCH 020/210] -- deleted depreced pip --- circle.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/circle.yml b/circle.yml index d9c321c9b77..d9fff638dea 100644 --- a/circle.yml +++ b/circle.yml @@ -15,6 +15,7 @@ machine: version: 0.10.33 dependencies: + pre: - "[ -e ~/.local/bin/circleci-matrix ] || mkdir -p ~/.local/bin @@ -22,8 +23,8 @@ dependencies: && chmod +x ~/.local/bin/circleci-matrix" override: - - pip install -r requirements.txt --allow-all-external - - pip install -r dev-requirements.txt --allow-all-external + - pip install -r requirements.txt + - pip install -r dev-requirements.txt - python setup.py develop post: @@ -38,6 +39,7 @@ dependencies: - ~/nvm/v0.10.33/bin/phantomjs database: + post: - sudo -E -u postgres ./bin/postgres_init/1_create_ckan_db.sh - sudo -E -u postgres ./bin/postgres_init/2_create_ckan_datastore_db.sh @@ -53,6 +55,7 @@ database: - paster db init -c test-core.ini test: + override: - circleci-matrix: parallel: true From b2bbdc6db56f0f5d9b72708279454127fe9ac387 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 19:55:43 +0200 Subject: [PATCH 021/210] Bleach Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index a675ac6bae4..f23ade27f18 100644 --- a/requirements.in +++ b/requirements.in @@ -4,7 +4,7 @@ Babel==2.3.4 Jinja2==2.8 Pylons==0.9.7 Beaker==1.7.0 # need to pin this because of https://github.com/bbangert/beaker/commit/fc511ceda4305fcf54919b3f081fed7790e2fef3 -bleach==1.4.2 +bleach==1.4.3 WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 fanstatic==0.12 Markdown==2.4 diff --git a/requirements.txt b/requirements.txt index 8357bd8c155..f21a215b53e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ argparse==1.4.0 # via ofs Babel==2.3.4 Beaker==1.7.0 -bleach==1.4.2 +bleach==1.4.3 decorator==4.0.6 # via pylons, sqlalchemy-migrate fanstatic==0.12 Flask==0.10.1 From 2e3f47b68cb9055fd6b74a578a7b4f332e2403f9 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 20:07:15 +0200 Subject: [PATCH 022/210] Mako Updated --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f21a215b53e..51f99c490b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,7 @@ FormEncode==1.3.0 html5lib==0.9999999 # via bleach itsdangerous==0.24 # via flask Jinja2==2.8 -Mako==1.0.3 +Mako==1.0.4 Markdown==2.4 MarkupSafe==0.23 nose==1.3.7 # via pylons From 5583f2a978459192f76fb4dcccc811711332ece3 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 20:15:51 +0200 Subject: [PATCH 023/210] Markdown Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index f23ade27f18..4985bdc93f4 100644 --- a/requirements.in +++ b/requirements.in @@ -7,7 +7,7 @@ Beaker==1.7.0 # need to pin this because of https://github.com/bbangert/beaker/c bleach==1.4.3 WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 fanstatic==0.12 -Markdown==2.4 +Markdown==2.6.6 ofs==0.4.1 ordereddict==1.1 Pairtree==0.7.1-T diff --git a/requirements.txt b/requirements.txt index 51f99c490b2..7f6cf90da2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ html5lib==0.9999999 # via bleach itsdangerous==0.24 # via flask Jinja2==2.8 Mako==1.0.4 -Markdown==2.4 +Markdown==2.6.6 MarkupSafe==0.23 nose==1.3.7 # via pylons ofs==0.4.1 From 5f824d3b3fdd4c4df5773ba907077f4bbad99cfe Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 20:25:51 +0200 Subject: [PATCH 024/210] OFS Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 4985bdc93f4..779c55d00c8 100644 --- a/requirements.in +++ b/requirements.in @@ -8,7 +8,7 @@ bleach==1.4.3 WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 fanstatic==0.12 Markdown==2.6.6 -ofs==0.4.1 +ofs==0.4.2 ordereddict==1.1 Pairtree==0.7.1-T paste==1.7.5.1 diff --git a/requirements.txt b/requirements.txt index 7f6cf90da2e..3f912fd35cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,7 +19,7 @@ Mako==1.0.4 Markdown==2.6.6 MarkupSafe==0.23 nose==1.3.7 # via pylons -ofs==0.4.1 +ofs==0.4.2 ordereddict==1.1 Pairtree==0.7.1-T passlib==1.6.2 From 181addb54fe9be18078797fe6fdf7c160d6726b9 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 20:34:21 +0200 Subject: [PATCH 025/210] PairTree Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 779c55d00c8..81720f28d73 100644 --- a/requirements.in +++ b/requirements.in @@ -10,7 +10,7 @@ fanstatic==0.12 Markdown==2.6.6 ofs==0.4.2 ordereddict==1.1 -Pairtree==0.7.1-T +Pairtree==0.5.8 paste==1.7.5.1 passlib==1.6.2 psycopg2==2.4.5 diff --git a/requirements.txt b/requirements.txt index 3f912fd35cc..f6cd2fb8ee8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ MarkupSafe==0.23 nose==1.3.7 # via pylons ofs==0.4.2 ordereddict==1.1 -Pairtree==0.7.1-T +Pairtree==0.5.8 passlib==1.6.2 paste==1.7.5.1 PasteDeploy==1.5.2 From a21f3f723719e8f433c3f4419da85592222f7669 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 20:44:55 +0200 Subject: [PATCH 026/210] Passlib Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 81720f28d73..1deef37ce65 100644 --- a/requirements.in +++ b/requirements.in @@ -12,7 +12,7 @@ ofs==0.4.2 ordereddict==1.1 Pairtree==0.5.8 paste==1.7.5.1 -passlib==1.6.2 +passlib==1.6.5 psycopg2==2.4.5 pysolr==3.4.0 python-dateutil>=1.5.0,<2.0.0 diff --git a/requirements.txt b/requirements.txt index f6cd2fb8ee8..0d31bc9ebf7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ nose==1.3.7 # via pylons ofs==0.4.2 ordereddict==1.1 Pairtree==0.5.8 -passlib==1.6.2 +passlib==1.6.5 paste==1.7.5.1 PasteDeploy==1.5.2 PasteScript==2.0.2 From 06fb95c023a3685df1cfaa1aa0c1b92763ae3e50 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 21:18:05 +0200 Subject: [PATCH 027/210] Pygments Updated --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0d31bc9ebf7..bb51516856c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,7 +29,7 @@ PasteScript==2.0.2 pbr==0.11.1 # via sqlalchemy-migrate psycopg2==2.4.5 pysolr==3.4.0 -Pygments==2.1 +Pygments==2.1.3 Pylons==0.9.7 python-dateutil==1.5 pytz==2016.4 From f21388b5e2b8bc773e733e51bfac0f045d2f81a6 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 21:19:25 +0200 Subject: [PATCH 028/210] PySolr Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 1deef37ce65..c530f0d4c54 100644 --- a/requirements.in +++ b/requirements.in @@ -14,7 +14,7 @@ Pairtree==0.5.8 paste==1.7.5.1 passlib==1.6.5 psycopg2==2.4.5 -pysolr==3.4.0 +pysolr==3.5.0 python-dateutil>=1.5.0,<2.0.0 pyutilib.component.core==4.5.3 repoze.who-friendlyform==1.0.8 diff --git a/requirements.txt b/requirements.txt index bb51516856c..322a011ab84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -28,7 +28,7 @@ PasteDeploy==1.5.2 PasteScript==2.0.2 pbr==0.11.1 # via sqlalchemy-migrate psycopg2==2.4.5 -pysolr==3.4.0 +pysolr==3.5.0 Pygments==2.1.3 Pylons==0.9.7 python-dateutil==1.5 From 6aec9ed1102fa070cadca9401cd9130c069fa72d Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 21:37:22 +0200 Subject: [PATCH 029/210] PyUtilLib Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index c530f0d4c54..b87ea9345dc 100644 --- a/requirements.in +++ b/requirements.in @@ -16,7 +16,7 @@ passlib==1.6.5 psycopg2==2.4.5 pysolr==3.5.0 python-dateutil>=1.5.0,<2.0.0 -pyutilib.component.core==4.5.3 +pyutilib.component.core==4.6.4 repoze.who-friendlyform==1.0.8 repoze.who==2.0 requests==2.7.0 diff --git a/requirements.txt b/requirements.txt index 322a011ab84..e087653f2b5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,7 +33,7 @@ Pygments==2.1.3 Pylons==0.9.7 python-dateutil==1.5 pytz==2016.4 -pyutilib.component.core==4.5.3 +pyutilib.component.core==4.6.4 repoze.lru==0.6 # via routes repoze.who-friendlyform==1.0.8 repoze.who==2.0 From e10700f7f37d5a5fcb665f9ccbf2613cf4224f4e Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 21:45:38 +0200 Subject: [PATCH 030/210] Repose.who Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index b87ea9345dc..1180bd0c078 100644 --- a/requirements.in +++ b/requirements.in @@ -18,7 +18,7 @@ pysolr==3.5.0 python-dateutil>=1.5.0,<2.0.0 pyutilib.component.core==4.6.4 repoze.who-friendlyform==1.0.8 -repoze.who==2.0 +repoze.who==2.3 requests==2.7.0 Routes==1.13 sqlalchemy-migrate==0.9.1 diff --git a/requirements.txt b/requirements.txt index e087653f2b5..d86233858a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ pytz==2016.4 pyutilib.component.core==4.6.4 repoze.lru==0.6 # via routes repoze.who-friendlyform==1.0.8 -repoze.who==2.0 +repoze.who==2.3 requests==2.7.0 Routes==1.13 simplejson==3.3.1 # via pylons HAND-FIXED FOR NOW #2681 From 84fba05e1f8b0353ebc1108068aee6280998235c Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 22:17:22 +0200 Subject: [PATCH 031/210] Requests Update --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 1180bd0c078..6713b3145fc 100644 --- a/requirements.in +++ b/requirements.in @@ -19,7 +19,7 @@ python-dateutil>=1.5.0,<2.0.0 pyutilib.component.core==4.6.4 repoze.who-friendlyform==1.0.8 repoze.who==2.3 -requests==2.7.0 +requests==2.10.0 Routes==1.13 sqlalchemy-migrate==0.9.1 SQLAlchemy==0.9.6 diff --git a/requirements.txt b/requirements.txt index d86233858a8..d35d3f49ccf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,7 +37,7 @@ pyutilib.component.core==4.6.4 repoze.lru==0.6 # via routes repoze.who-friendlyform==1.0.8 repoze.who==2.3 -requests==2.7.0 +requests==2.10.0 Routes==1.13 simplejson==3.3.1 # via pylons HAND-FIXED FOR NOW #2681 six==1.10.0 # via bleach, html5lib, pastescript, sqlalchemy-migrate From 5fbb580e68bbbe8fcf2e0e9c0eccaf26f8d9a86e Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 22:23:55 +0200 Subject: [PATCH 032/210] SimpleJson Updated --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d35d3f49ccf..a0fbf19c180 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ repoze.who-friendlyform==1.0.8 repoze.who==2.3 requests==2.10.0 Routes==1.13 -simplejson==3.3.1 # via pylons HAND-FIXED FOR NOW #2681 +simplejson==3.8.2 # via pylons HAND-FIXED FOR NOW #2681 six==1.10.0 # via bleach, html5lib, pastescript, sqlalchemy-migrate sqlalchemy-migrate==0.9.1 SQLAlchemy==0.9.6 From 4c4c8974a3bf05ff53a1b0971ed0fb7901af0477 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 22:33:51 +0200 Subject: [PATCH 033/210] SqlParse Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 6713b3145fc..8ecdfdfe201 100644 --- a/requirements.in +++ b/requirements.in @@ -24,7 +24,7 @@ Routes==1.13 sqlalchemy-migrate==0.9.1 SQLAlchemy==0.9.6 vdm==0.13 -sqlparse==0.1.11 +sqlparse==0.1.19 WebHelpers==1.3 WebOb==1.0.8 zope.interface==4.1.1 diff --git a/requirements.txt b/requirements.txt index a0fbf19c180..9fd42b37c47 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,7 +43,7 @@ simplejson==3.8.2 # via pylons HAND-FIXED FOR NOW #2681 six==1.10.0 # via bleach, html5lib, pastescript, sqlalchemy-migrate sqlalchemy-migrate==0.9.1 SQLAlchemy==0.9.6 -sqlparse==0.1.11 +sqlparse==0.1.19 Tempita==0.5.2 tzlocal==1.2.2 unicodecsv==0.14.1 From b08008058213286363cb90a74ee0e26a722f9e29 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 22:40:15 +0200 Subject: [PATCH 034/210] WebError Upgrade --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9fd42b37c47..0bf20c00641 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,7 +48,7 @@ Tempita==0.5.2 tzlocal==1.2.2 unicodecsv==0.14.1 vdm==0.13 -WebError==0.11 +WebError==0.13.1 WebHelpers==1.3 WebOb==1.0.8 WebTest==1.4.3 From 635d15f100d61cabba84bc8244c48857a0347025 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 22:50:43 +0200 Subject: [PATCH 035/210] SqlAlchemy-migrate Updated --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 8ecdfdfe201..23be1be4150 100644 --- a/requirements.in +++ b/requirements.in @@ -21,7 +21,7 @@ repoze.who-friendlyform==1.0.8 repoze.who==2.3 requests==2.10.0 Routes==1.13 -sqlalchemy-migrate==0.9.1 +sqlalchemy-migrate==0.10.0 SQLAlchemy==0.9.6 vdm==0.13 sqlparse==0.1.19 diff --git a/requirements.txt b/requirements.txt index 0bf20c00641..fe39fb35929 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,7 +41,7 @@ requests==2.10.0 Routes==1.13 simplejson==3.8.2 # via pylons HAND-FIXED FOR NOW #2681 six==1.10.0 # via bleach, html5lib, pastescript, sqlalchemy-migrate -sqlalchemy-migrate==0.9.1 +sqlalchemy-migrate==0.10.0 SQLAlchemy==0.9.6 sqlparse==0.1.19 Tempita==0.5.2 From 8dda8c471df48e70ea99083b010830a7cdf86a4f Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 22:58:10 +0200 Subject: [PATCH 036/210] Werkzeug Upgrade --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fe39fb35929..0f06aa5b030 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,7 +52,7 @@ WebError==0.13.1 WebHelpers==1.3 WebOb==1.0.8 WebTest==1.4.3 -Werkzeug==0.11.3 +Werkzeug==0.11.10 wsgi-party==0.1b1 zope.interface==4.1.1 From b83159a02dbf7bbdf9f0ddfc371c21db58b71ca2 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 23:03:38 +0200 Subject: [PATCH 037/210] zope.interface Upgraded --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 23be1be4150..16fbe03fd47 100644 --- a/requirements.in +++ b/requirements.in @@ -27,7 +27,7 @@ vdm==0.13 sqlparse==0.1.19 WebHelpers==1.3 WebOb==1.0.8 -zope.interface==4.1.1 +zope.interface==4.2.0 unicodecsv>=0.9 pytz==2016.4 tzlocal==1.2.2 diff --git a/requirements.txt b/requirements.txt index 0f06aa5b030..9e394c995f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -54,7 +54,7 @@ WebOb==1.0.8 WebTest==1.4.3 Werkzeug==0.11.10 wsgi-party==0.1b1 -zope.interface==4.1.1 +zope.interface==4.2.0 # The following packages are commented out because they are # considered to be unsafe in a requirements file: From 359f8c22fd5e281b4c9d1410d85b7614f78829f1 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 23:24:59 +0200 Subject: [PATCH 038/210] Requirements.in - Ordered by name --- requirements.in | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/requirements.in b/requirements.in index 16fbe03fd47..257251e5f08 100644 --- a/requirements.in +++ b/requirements.in @@ -1,21 +1,22 @@ # The file contains the direct ckan requirements. # Use pip-compile to create a requirements.txt file from this Babel==2.3.4 -Jinja2==2.8 -Pylons==0.9.7 Beaker==1.7.0 # need to pin this because of https://github.com/bbangert/beaker/commit/fc511ceda4305fcf54919b3f081fed7790e2fef3 bleach==1.4.3 -WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 fanstatic==0.12 +Flask==0.10.1 +Jinja2==2.8 Markdown==2.6.6 ofs==0.4.2 ordereddict==1.1 Pairtree==0.5.8 -paste==1.7.5.1 passlib==1.6.5 +paste==1.7.5.1 psycopg2==2.4.5 pysolr==3.5.0 +Pylons==0.9.7 python-dateutil>=1.5.0,<2.0.0 +pytz==2016.4 pyutilib.component.core==4.6.4 repoze.who-friendlyform==1.0.8 repoze.who==2.3 @@ -23,13 +24,12 @@ requests==2.10.0 Routes==1.13 sqlalchemy-migrate==0.10.0 SQLAlchemy==0.9.6 -vdm==0.13 sqlparse==0.1.19 +tzlocal==1.2.2 +unicodecsv>=0.9 +vdm==0.13 WebHelpers==1.3 WebOb==1.0.8 -zope.interface==4.2.0 -unicodecsv>=0.9 -pytz==2016.4 -tzlocal==1.2.2 +WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 wsgi-party==0.1b1 -Flask==0.10.1 +zope.interface==4.2.0 From 6f1e8e7d87b89b05ec78b36d63a075d109b899d4 Mon Sep 17 00:00:00 2001 From: Deinok Date: Sun, 12 Jun 2016 23:27:58 +0200 Subject: [PATCH 039/210] Added Requires.io bandge It shows the outdated requirements --- README.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.rst b/README.rst index 6f45f30f68a..2fadc2dd390 100644 --- a/README.rst +++ b/README.rst @@ -20,6 +20,10 @@ CKAN: The Open Source Data Portal Software :target: https://coveralls.io/github/ckan/ckan?branch=master :alt: Coverage Status +.. image:: https://requires.io/github/ckan/ckan/requirements.svg?branch=master + :target: https://requires.io/github/ckan/ckan/requirements/?branch=master + :alt: Requirements Status + **CKAN is the world’s leading open-source data portal platform**. CKAN makes it easy to publish, share and work with data. It's a data management system that provides a powerful platform for cataloging, storing and accessing From 2a5e5ea767dd108a4f7fe462e4a70d6faa5f199b Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 13 Jun 2016 09:48:11 +0200 Subject: [PATCH 040/210] Sorted Dev-requirements by name --- dev-requirements.txt | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8e8a389c3b8..e6468b7fc66 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,15 +1,19 @@ # These are packages that required when running ckan tests and building the docs +beautifulsoup4==4.3.2 +coveralls docutils==0.8.1 +factory-boy==2.1.1 httpretty==0.8.3 -# nose==1.3.0 # already in requirements.txt -pep8==1.4.6 -Sphinx==1.2.3 -polib==1.0.3 mock==1.0.1 -factory-boy==2.1.1 -coveralls -sphinx-rtd-theme==0.1.9 -beautifulsoup4==4.3.2 +pep8==1.4.6 pip-tools==1.1.2 +polib==1.0.3 pyfakefs==2.7 +Sphinx==1.2.3 +sphinx-rtd-theme==0.1.9 + +# nose==1.3.0 # already in requirements.txt + + + From 9863285ec4016afb873adbe7a95aaa4f0773e927 Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 13 Jun 2016 09:48:46 +0200 Subject: [PATCH 041/210] Coveralls Pined --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index e6468b7fc66..607d9c40d1b 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # These are packages that required when running ckan tests and building the docs beautifulsoup4==4.3.2 -coveralls +coveralls==1.1 docutils==0.8.1 factory-boy==2.1.1 httpretty==0.8.3 From 0eaca7924b405a1f519a2100f0a157012242b4d8 Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 13 Jun 2016 10:07:13 +0200 Subject: [PATCH 042/210] BeautifulSoup Updated --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 607d9c40d1b..4047609ccec 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,6 +1,6 @@ # These are packages that required when running ckan tests and building the docs -beautifulsoup4==4.3.2 +beautifulsoup4==4.4.1 coveralls==1.1 docutils==0.8.1 factory-boy==2.1.1 From a116078fec1e68f91be995aeafe5a9dc02ef28ce Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 13 Jun 2016 10:58:58 +0200 Subject: [PATCH 043/210] DocUtils Updated --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 4047609ccec..83291f4e01a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ beautifulsoup4==4.4.1 coveralls==1.1 -docutils==0.8.1 +docutils==0.12 factory-boy==2.1.1 httpretty==0.8.3 mock==1.0.1 From 99c0f33b7feb33c8c4d6e3ed40a26dc84a7f3e51 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 13 Jun 2016 10:21:53 +0100 Subject: [PATCH 044/210] [#3108] Raise validation errors on group/org_member_create Add not_missing to the schema and raise the errors on the action. Adds some tests as well --- ckan/logic/action/create.py | 3 ++ ckan/logic/schema.py | 6 ++-- ckan/tests/logic/action/test_create.py | 43 +++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 4 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 264fe0ca2ee..0a2a175309d 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -1372,6 +1372,9 @@ def _group_or_org_member_create(context, data_dict, is_org=False): schema = ckan.logic.schema.member_schema() data, errors = _validate(data_dict, schema, context) + if errors: + model.Session.rollback() + raise ValidationError(errors) username = _get_or_bust(data_dict, 'username') role = data_dict.get('role') diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index b16c914f28a..ad1f3d8cebe 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -532,9 +532,9 @@ def default_follow_dataset_schema(): def member_schema(): schema = { - 'id': [group_id_exists, unicode], - 'username': [user_name_exists, unicode], - 'role': [role_exists, unicode], + 'id': [not_missing, not_empty, group_id_exists, unicode], + 'username': [not_missing, user_name_exists, unicode], + 'role': [not_missing, role_exists, unicode], } return schema diff --git a/ckan/tests/logic/action/test_create.py b/ckan/tests/logic/action/test_create.py index b48f65c5f9d..e38ab5656fd 100644 --- a/ckan/tests/logic/action/test_create.py +++ b/ckan/tests/logic/action/test_create.py @@ -151,7 +151,6 @@ def setup_class(cls): @classmethod def teardown_class(cls): - p.unload('image_view') helpers.reset_db() @@ -485,6 +484,48 @@ def test_organization_member_creation(self): assert_equals(new_membership['table_id'], user['id']) assert_equals(new_membership['capacity'], 'member') + def test_group_member_creation_raises_validation_error_if_id_missing(self): + + assert_raises(logic.ValidationError, + helpers.call_action, 'group_member_create', + username='someuser', + role='member',) + + def test_group_member_creation_raises_validation_error_if_username_missing(self): + + assert_raises(logic.ValidationError, + helpers.call_action, 'group_member_create', + id='someid', + role='member',) + + def test_group_member_creation_raises_validation_error_if_role_missing(self): + + assert_raises(logic.ValidationError, + helpers.call_action, 'group_member_create', + id='someid', + username='someuser',) + + def test_org_member_creation_raises_validation_error_if_id_missing(self): + + assert_raises(logic.ValidationError, + helpers.call_action, 'organization_member_create', + username='someuser', + role='member',) + + def test_org_member_creation_raises_validation_error_if_username_missing(self): + + assert_raises(logic.ValidationError, + helpers.call_action, 'organization_member_create', + id='someid', + role='member',) + + def test_org_member_creation_raises_validation_error_if_role_missing(self): + + assert_raises(logic.ValidationError, + helpers.call_action, 'organization_member_create', + id='someid', + username='someuser',) + class TestDatasetCreate(helpers.FunctionalTestBase): From b3354c0339f2f54e38840ff0fe8d0470297dd18b Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 13 Jun 2016 11:30:36 +0200 Subject: [PATCH 045/210] Mock Updated --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 83291f4e01a..9faef5cef58 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -5,7 +5,7 @@ coveralls==1.1 docutils==0.12 factory-boy==2.1.1 httpretty==0.8.3 -mock==1.0.1 +mock==2.0.0 pep8==1.4.6 pip-tools==1.1.2 polib==1.0.3 From f0798fa84f62414d94ec16c6e385cd0aa71e00a1 Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 13 Jun 2016 11:53:14 +0200 Subject: [PATCH 046/210] Pip-Tools Updated --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 9faef5cef58..5a294b0fb37 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,7 +7,7 @@ factory-boy==2.1.1 httpretty==0.8.3 mock==2.0.0 pep8==1.4.6 -pip-tools==1.1.2 +pip-tools==1.6.5 polib==1.0.3 pyfakefs==2.7 Sphinx==1.2.3 From b2778c82fb901891b95b899ae0847547c5e53189 Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 13 Jun 2016 12:00:45 +0200 Subject: [PATCH 047/210] PoLib Updated --- dev-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 5a294b0fb37..3d4f0e07943 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,7 +8,7 @@ httpretty==0.8.3 mock==2.0.0 pep8==1.4.6 pip-tools==1.6.5 -polib==1.0.3 +polib==1.0.7 pyfakefs==2.7 Sphinx==1.2.3 sphinx-rtd-theme==0.1.9 From 1f23b9746ba25d1a227705078f111368bb8758a0 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 13 Jun 2016 11:59:29 +0100 Subject: [PATCH 048/210] [#3108] Allow to renferece groups and users by id or name --- ckan/logic/schema.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index ad1f3d8cebe..5556b40b1e9 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -55,8 +55,10 @@ resource_id_exists, tag_not_in_vocabulary, group_id_exists, + group_id_or_name_exists, owner_org_validator, user_name_exists, + user_id_or_name_exists, role_exists, datasets_with_no_organization_cannot_be_private, list_of_strings, @@ -532,8 +534,8 @@ def default_follow_dataset_schema(): def member_schema(): schema = { - 'id': [not_missing, not_empty, group_id_exists, unicode], - 'username': [not_missing, user_name_exists, unicode], + 'id': [not_missing, not_empty, group_id_or_name_exists, unicode], + 'username': [not_missing, user_id_or_name_exists, unicode], 'role': [not_missing, role_exists, unicode], } return schema From fde822fefe350238910d6f6bea51d188d240e0a1 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Mon, 13 Jun 2016 16:33:18 +0200 Subject: [PATCH 049/210] Docs: Extensions should not automatically alter the database structure. See https://github.com/ckan/ideas-and-roadmap/issues/164. --- doc/extensions/best-practices.rst | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/extensions/best-practices.rst b/doc/extensions/best-practices.rst index 078fbc84588..422dc50bec2 100644 --- a/doc/extensions/best-practices.rst +++ b/doc/extensions/best-practices.rst @@ -67,12 +67,28 @@ adding them to ``setup.py``, they should be added to ``requirements.txt``, which can be installed with:: pip install -r requirements.txt - + To prevent accidental breakage of your extension through backwards-incompatible behaviour of newer versions of your dependencies, their versions should be pinned, such as:: requests==2.7.0 - + On the flip side, be mindful that this could also create version conflicts with requirements of considerably newer or older extensions. + + +-------------------------------------------------- +Do not automatically modify the database structure +-------------------------------------------------- + +If your extension uses custom database tables then it needs to modify the +database structure, for example to add the tables after its installation or to +migrate them after an update. These modifications should not be performed +automatically when the extension is loaded, since this can lead to `dead-locks +and other problems`_. + +Instead, create a :doc:`paster command ` which can be run separately. + +.. _dead-locks and other problems: https://github.com/ckan/ideas-and-roadmap/issues/164 + From 23e0ea31ef77725e76891e3bde049721f11d9c76 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Mon, 13 Jun 2016 16:50:22 +0200 Subject: [PATCH 050/210] Update extension template to reflect best practice w.r.t. requirements. Extensions should use a separate ``requirements.txt`` file to list their dependencies (instead of ``setup.py``). This was already documented but the ``setup.py`` template did not mention it. --- ckan/pastertemplates/template/MANIFEST.in_tmpl | 2 ++ .../template/requirements.txt_tmpl | 0 ckan/pastertemplates/template/setup.py_tmpl | 16 +++++++++------- 3 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 ckan/pastertemplates/template/requirements.txt_tmpl diff --git a/ckan/pastertemplates/template/MANIFEST.in_tmpl b/ckan/pastertemplates/template/MANIFEST.in_tmpl index dc63f8f739d..204b4d35dc8 100644 --- a/ckan/pastertemplates/template/MANIFEST.in_tmpl +++ b/ckan/pastertemplates/template/MANIFEST.in_tmpl @@ -1,2 +1,4 @@ include README.rst +include LICENSE +include requirements.txt recursive-include ckanext/{{ project_shortname }} *.html *.json *.js *.less *.css *.mo diff --git a/ckan/pastertemplates/template/requirements.txt_tmpl b/ckan/pastertemplates/template/requirements.txt_tmpl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ckan/pastertemplates/template/setup.py_tmpl b/ckan/pastertemplates/template/setup.py_tmpl index 921035fefd2..844042b0035 100644 --- a/ckan/pastertemplates/template/setup.py_tmpl +++ b/ckan/pastertemplates/template/setup.py_tmpl @@ -56,11 +56,12 @@ setup( packages=find_packages(exclude=['contrib', 'docs', 'tests*']), namespace_packages=['ckanext'], - # List run-time dependencies here. These will be installed by pip when your - # project is installed. For an analysis of "install_requires" vs pip's - # requirements files see: - # https://packaging.python.org/en/latest/technical.html#install-requires-vs-requirements-files - install_requires=[], + install_requires=[ + # CKAN extensions should not list dependencies here, but in a separate + # ``requirements.txt`` file. + # + # http://docs.ckan.org/en/latest/extensions/best-practices.html#add-third-party-libraries-to-requirements-txt + ], # If there are data files included in your packages that need to be # installed, specify them here. If using Python 2.6 or less, then these @@ -81,8 +82,9 @@ setup( entry_points=''' [ckan.plugins] {{ project_shortname }}=ckanext.{{ project_shortname }}.plugin:{{ plugin_class_name }} - [babel.extractors] - ckan = ckan.lib.extract:extract_ckan + + [babel.extractors] + ckan = ckan.lib.extract:extract_ckan ''', # If you are changing from the default layout of your extension, you may From d36226b443f30cad4b70f6b8a881cb0066945b6b Mon Sep 17 00:00:00 2001 From: Martin Burchell Date: Mon, 13 Jun 2016 17:48:44 +0100 Subject: [PATCH 051/210] Purge dataset relationships --- ckan/logic/action/delete.py | 7 +++++++ ckan/tests/logic/action/test_delete.py | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index 87f9050a4d1..fe4300a2ef7 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -117,6 +117,8 @@ def dataset_purge(context, data_dict): :type id: string ''' + from sqlalchemy import or_ + model = context['model'] id = _get_or_bust(data_dict, 'id') @@ -134,6 +136,11 @@ def dataset_purge(context, data_dict): for m in members.all(): m.purge() + for r in model.Session.query(model.PackageRelationship).filter( + or_(model.PackageRelationship.subject_package_id == pkg.id, + model.PackageRelationship.object_package_id == pkg.id)).all(): + r.purge() + pkg = model.Package.get(id) # no new_revision() needed since there are no object_revisions created # during purge diff --git a/ckan/tests/logic/action/test_delete.py b/ckan/tests/logic/action/test_delete.py index 94b750a8ff2..ad9eba3ab90 100644 --- a/ckan/tests/logic/action/test_delete.py +++ b/ckan/tests/logic/action/test_delete.py @@ -451,6 +451,30 @@ def test_purged_dataset_leaves_no_trace_in_the_model(self): # No Revision objects were purged or created assert_equals(num_revisions_after - num_revisions_before, 0) + def test_purged_dataset_removed_from_relationships(self): + child = factories.Dataset() + parent = factories.Dataset() + grandparent = factories.Dataset() + + helpers.call_action('package_relationship_create', + subject=child['id'], + type='child_of', + object=parent['id']) + + helpers.call_action('package_relationship_create', + subject=parent['id'], + type='child_of', + object=grandparent['id']) + + assert_equals(len( + model.Session.query(model.PackageRelationship).all()), 2) + + helpers.call_action('dataset_purge', + context={'ignore_auth': True}, + id=parent['name']) + + assert_equals(model.Session.query(model.PackageRelationship).all(), []) + def test_missing_id_returns_error(self): assert_raises(logic.ValidationError, helpers.call_action, 'dataset_purge') From 688a1a769becca06fb115edd3467781dd1f62de4 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Tue, 14 Jun 2016 12:02:14 +0100 Subject: [PATCH 052/210] [#3113] Remove reference to lang in session. lang maintained by i18n middleware, not by key in cookie. This continues removal of old session code missed in #3053. --- ckan/lib/base.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 72446b366be..b18c89e0d85 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -309,16 +309,9 @@ def __call__(self, environ, start_response): is_valid_cookie_data = True break if not is_valid_cookie_data: - if session.id: - if not session.get('lang'): - self.log.debug('No session data any more - ' - 'deleting session') - self.log.debug('Session: %r', session.items()) - session.delete() - else: - response.delete_cookie(cookie) - self.log.debug('No session data any more - ' - 'deleting session cookie') + response.delete_cookie(cookie) + self.log.debug('No session data any more - ' + 'deleting session cookie') # Remove auth_tkt repoze.who cookie if user not logged in. elif cookie == 'auth_tkt' and not session.id: response.delete_cookie(cookie) From ad8db409e48e4ccc6d3da7eb9a27d48ce586bad5 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Tue, 14 Jun 2016 12:17:13 +0100 Subject: [PATCH 053/210] [#3113] Restore session deletion. I over-committed in the previous commit and removed too much code. This restores session deletion if there is a session.id, but no valid session data. --- ckan/lib/base.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index b18c89e0d85..86b10c1399c 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -309,9 +309,15 @@ def __call__(self, environ, start_response): is_valid_cookie_data = True break if not is_valid_cookie_data: - response.delete_cookie(cookie) - self.log.debug('No session data any more - ' - 'deleting session cookie') + if session.id: + self.log.debug('No valid session data - ' + 'deleting session') + self.log.debug('Session: %r', session.items()) + session.delete() + else: + self.log.debug('No session id - ' + 'deleting session cookie') + response.delete_cookie(cookie) # Remove auth_tkt repoze.who cookie if user not logged in. elif cookie == 'auth_tkt' and not session.id: response.delete_cookie(cookie) From 7528a736623ff58c14347263ae422cbd4efc3f67 Mon Sep 17 00:00:00 2001 From: Deinok Date: Tue, 14 Jun 2016 15:11:13 +0200 Subject: [PATCH 054/210] Coveralls Unpinned - Added Comment --- dev-requirements.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 3d4f0e07943..095abe8a547 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ # These are packages that required when running ckan tests and building the docs beautifulsoup4==4.4.1 -coveralls==1.1 +coveralls #Let Unpinned - Requires latest coveralls docutils==0.12 factory-boy==2.1.1 httpretty==0.8.3 @@ -14,6 +14,3 @@ Sphinx==1.2.3 sphinx-rtd-theme==0.1.9 # nose==1.3.0 # already in requirements.txt - - - From e20bef268bd3108642bddd328a2ad84f2cae59e4 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 15 Jun 2016 10:08:33 +0100 Subject: [PATCH 055/210] [#3108] Add back plugin unload command --- ckan/tests/logic/action/test_create.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/logic/action/test_create.py b/ckan/tests/logic/action/test_create.py index e38ab5656fd..37cbce676eb 100644 --- a/ckan/tests/logic/action/test_create.py +++ b/ckan/tests/logic/action/test_create.py @@ -151,7 +151,7 @@ def setup_class(cls): @classmethod def teardown_class(cls): - + p.unload('image_view') helpers.reset_db() def setup(self): From 83f4a477c238965ae76035643747b995a103e6d6 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 15 Jun 2016 10:12:27 +0100 Subject: [PATCH 056/210] [#3108] Remove redundant validator --- ckan/logic/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 5556b40b1e9..fa2cc484979 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -534,7 +534,7 @@ def default_follow_dataset_schema(): def member_schema(): schema = { - 'id': [not_missing, not_empty, group_id_or_name_exists, unicode], + 'id': [not_missing, group_id_or_name_exists, unicode], 'username': [not_missing, user_id_or_name_exists, unicode], 'role': [not_missing, role_exists, unicode], } From d9f3a5036a99c5e447ad93977e99ba745e2868c3 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Wed, 15 Jun 2016 17:19:58 +0100 Subject: [PATCH 057/210] [#3116] Refactor middleware module. It was starting to become unmanageably large with a lot of separate concerns in one place. - Separate the flask and pylons app code into separate modules. - Separate the common middleware code into a separate module. --- ckan/config/middleware.py | 595 -------------------- ckan/config/middleware/__init__.py | 132 +++++ ckan/config/middleware/common_middleware.py | 216 +++++++ ckan/config/middleware/flask_app.py | 81 +++ ckan/config/middleware/pylons_app.py | 218 +++++++ ckan/tests/config/test_middleware.py | 3 +- 6 files changed, 649 insertions(+), 596 deletions(-) delete mode 100644 ckan/config/middleware.py create mode 100644 ckan/config/middleware/__init__.py create mode 100644 ckan/config/middleware/common_middleware.py create mode 100644 ckan/config/middleware/flask_app.py create mode 100644 ckan/config/middleware/pylons_app.py diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py deleted file mode 100644 index c9898fc11ae..00000000000 --- a/ckan/config/middleware.py +++ /dev/null @@ -1,595 +0,0 @@ -# encoding: utf-8 - -"""Pylons middleware initialization""" -import urllib -import urllib2 -import logging -import json -import hashlib -import os - -import sqlalchemy as sa -from beaker.middleware import CacheMiddleware, SessionMiddleware -from paste.cascade import Cascade -from paste.registry import RegistryManager -from paste.urlparser import StaticURLParser -from paste.deploy.converters import asbool -from pylons import config -from pylons.middleware import ErrorHandler, StatusCodeRedirect -from pylons.wsgiapp import PylonsApp -from routes.middleware import RoutesMiddleware -from repoze.who.config import WhoConfig -from repoze.who.middleware import PluggableAuthenticationMiddleware -from fanstatic import Fanstatic - -from wsgi_party import WSGIParty, HighAndDry -from flask import Flask -from flask import abort as flask_abort -from flask import request as flask_request -from flask import _request_ctx_stack -from werkzeug.exceptions import HTTPException -from werkzeug.test import create_environ, run_wsgi_app - -from ckan.plugins import PluginImplementations -from ckan.plugins.interfaces import IMiddleware -from ckan.lib.i18n import get_locales_from_config -import ckan.lib.uploader as uploader - -from ckan.config.environment import load_environment -import ckan.lib.app_globals as app_globals - -log = logging.getLogger(__name__) - - -def make_app(conf, full_stack=True, static_files=True, **app_conf): - - # :::TODO::: like the flask app, make the pylons app respond to invites at - # /__invite__/, and handle can_handle_request requests. - - pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) - flask_app = make_flask_stack(conf) - - app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, 'flask_app': flask_app}) - - return app - - -def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): - """Create a Pylons WSGI application and return it - - ``conf`` - The inherited configuration for this application. Normally from - the [DEFAULT] section of the Paste ini file. - - ``full_stack`` - Whether this application provides a full WSGI stack (by default, - meaning it handles its own exceptions and errors). Disable - full_stack when this application is "managed" by another WSGI - middleware. - - ``static_files`` - Whether this application serves its own static files; disable - when another web server is responsible for serving them. - - ``app_conf`` - The application's local configuration. Normally specified in - the [app:] section of the Paste ini file (where - defaults to main). - - """ - # Configure the Pylons environment - load_environment(conf, app_conf) - - # The Pylons WSGI app - app = PylonsApp() - # set pylons globals - app_globals.reset() - - for plugin in PluginImplementations(IMiddleware): - app = plugin.make_middleware(app, config) - - app = RootPathMiddleware(app, config) - - # Routing/Session/Cache Middleware - app = RoutesMiddleware(app, config['routes.map']) - # we want to be able to retrieve the routes middleware to be able to update - # the mapper. We store it in the pylons config to allow this. - config['routes.middleware'] = app - app = SessionMiddleware(app, config) - app = CacheMiddleware(app, config) - - # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) - # app = QueueLogMiddleware(app) - if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): - app = execute_on_completion(app, config, cleanup_pylons_response_string) - - # Fanstatic - if asbool(config.get('debug', False)): - fanstatic_config = { - 'versioning': True, - 'recompute_hashes': True, - 'minified': False, - 'bottom': True, - 'bundle': False, - } - else: - fanstatic_config = { - 'versioning': True, - 'recompute_hashes': False, - 'minified': True, - 'bottom': True, - 'bundle': True, - } - app = Fanstatic(app, **fanstatic_config) - - for plugin in PluginImplementations(IMiddleware): - try: - app = plugin.make_error_log_middleware(app, config) - except AttributeError: - log.critical('Middleware class {0} is missing the method' - 'make_error_log_middleware.'.format(plugin.__class__.__name__)) - - if asbool(full_stack): - # Handle Python exceptions - app = ErrorHandler(app, conf, **config['pylons.errorware']) - - # Display error documents for 400, 403, 404 status codes (and - # 500 when debug is disabled) - if asbool(config['debug']): - app = StatusCodeRedirect(app, [400, 403, 404]) - else: - app = StatusCodeRedirect(app, [400, 403, 404, 500]) - - # Initialize repoze.who - who_parser = WhoConfig(conf['here']) - who_parser.parse(open(app_conf['who.config_file'])) - - app = PluggableAuthenticationMiddleware( - app, - who_parser.identifiers, - who_parser.authenticators, - who_parser.challengers, - who_parser.mdproviders, - who_parser.request_classifier, - who_parser.challenge_decider, - logging.getLogger('repoze.who'), - logging.WARN, # ignored - who_parser.remote_user_key - ) - - # Establish the Registry for this application - app = RegistryManager(app) - - app = I18nMiddleware(app, config) - - if asbool(static_files): - # Serve static files - static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ - else int(config.get('ckan.static_max_age', 3600)) - - static_app = StaticURLParser(config['pylons.paths']['static_files'], - cache_max_age=static_max_age) - static_parsers = [static_app, app] - - storage_directory = uploader.get_storage_path() - if storage_directory: - path = os.path.join(storage_directory, 'storage') - try: - os.makedirs(path) - except OSError, e: - # errno 17 is file already exists - if e.errno != 17: - raise - - storage_app = StaticURLParser(path, cache_max_age=static_max_age) - static_parsers.insert(0, storage_app) - - # Configurable extra static file paths - extra_static_parsers = [] - for public_path in config.get('extra_public_paths', '').split(','): - if public_path.strip(): - extra_static_parsers.append( - StaticURLParser(public_path.strip(), - cache_max_age=static_max_age) - ) - app = Cascade(extra_static_parsers + static_parsers) - - # Page cache - if asbool(config.get('ckan.page_cache_enabled')): - app = PageCacheMiddleware(app, config) - - # Tracking - if asbool(config.get('ckan.tracking_enabled', 'false')): - app = TrackingMiddleware(app, config) - - return app - - -def make_flask_stack(conf): - """ This has to pass the flask app through all the same middleware that - Pylons used """ - - app = CKANFlask(__name__) - - @app.route('/hello', methods=['GET']) - def hello_world(): - return 'Hello World, this is served by Flask' - - @app.route('/hello', methods=['POST']) - def hello_world_post(): - return 'Hello World, this was posted to Flask' - - return app - - -class CKANFlask(Flask): - - '''Extend the Flask class with a special view to join the 'partyline' - established by AskAppDispatcherMiddleware. - - Also provide a 'can_handle_request' method. - ''' - - def __init__(self, import_name, *args, **kwargs): - super(CKANFlask, self).__init__(import_name, *args, **kwargs) - self.add_url_rule('/__invite__/', endpoint='partyline', - view_func=self.join_party) - self.partyline = None - self.partyline_connected = False - self.invitation_context = None - self.app_name = None # A label for the app handling this request - # (this app). - - def join_party(self, request=flask_request): - # Bootstrap, turn the view function into a 404 after registering. - if self.partyline_connected: - # This route does not exist at the HTTP level. - flask_abort(404) - self.invitation_context = _request_ctx_stack.top - self.partyline = request.environ.get(WSGIParty.partyline_key) - self.app_name = request.environ.get('partyline_handling_app') - self.partyline.connect('can_handle_request', self.can_handle_request) - self.partyline_connected = True - return 'ok' - - def can_handle_request(self, environ): - ''' - Decides whether it can handle a request with the Flask app by - matching the request environ against the route mapper - - Returns (True, 'flask_app') if this is the case. - ''' - - # TODO: identify matching urls as core or extension. This will depend - # on how we setup routing in Flask - - urls = self.url_map.bind_to_environ(environ) - try: - endpoint, args = urls.match() - log.debug('Flask route match, endpoint: {0}, args: {1}'.format( - endpoint, args)) - return (True, self.app_name) - except HTTPException: - raise HighAndDry() - - -class AskAppDispatcherMiddleware(WSGIParty): - - ''' - Establish a 'partyline' to each provided app. Select which app to call - by asking each if they can handle the requested path at PATH_INFO. - - Used to help transition from Pylons to Flask, and should be removed once - Pylons has been deprecated and all app requests are handled by Flask. - - Each app should handle a call to 'can_handle_request(environ)', responding - with a tuple: - (, , []) - where: - `bool` is True if the app can handle the payload url, - `app` is the wsgi app returning the answer - `origin` is an optional string to determine where in the app the url - will be handled, e.g. 'core' or 'extension'. - - Order of precedence if more than one app can handle a url: - Flask Extension > Pylons Extension > Flask Core > Pylons Core - ''' - - def __init__(self, apps=None, invites=(), ignore_missing_services=False): - # Dict of apps managed by this middleware {: , ...} - self.apps = apps or {} - - # A dict of service name => handler mappings. - self.handlers = {} - - # If True, suppress :class:`NoSuchServiceName` errors. Default: False. - self.ignore_missing_services = ignore_missing_services - - self.send_invitations(apps) - - def send_invitations(self, apps): - '''Call each app at the invite route to establish a partyline. Called - on init.''' - PATH = '/__invite__/' - for app_name, app in apps.items(): - environ = create_environ(path=PATH) - environ[self.partyline_key] = self.operator_class(self) - # A reference to the handling app. Used to id the app when - # responding to a handling request. - environ['partyline_handling_app'] = app_name - run_wsgi_app(app, environ) - - def __call__(self, environ, start_response): - '''Determine which app to call by asking each app if it can handle the - url and method defined on the eviron''' - # :::TODO::: Enforce order of precedence for dispatching to apps here. - - app_name = 'pylons_app' # currently defaulting to pylons app - answers = self.ask_around('can_handle_request', environ) - log.debug('Route support answers for {0} {1}: {2}'.format( - environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), - answers)) - available_handlers = [] - for answer in answers: - if len(answer) == 2: - can_handle, asked_app = answer - origin = 'core' - else: - can_handle, asked_app, origin = answer - if can_handle: - available_handlers.append('{0}_{1}'.format(asked_app, origin)) - - # Enforce order of precedence: - # Flask Extension > Pylons Extension > Flask Core > Pylons Core - if available_handlers: - if 'flask_app_extension' in available_handlers: - app_name = 'flask_app' - elif 'pylons_app_extension' in available_handlers: - app_name = 'pylons_app' - elif 'flask_app_core' in available_handlers: - app_name = 'flask_app' - - log.debug('Serving request via {0} app'.format(app_name)) - environ['ckan.app'] = app_name - return self.apps[app_name](environ, start_response) - - -class RootPathMiddleware(object): - ''' - Prevents the SCRIPT_NAME server variable conflicting with the ckan.root_url - config. The routes package uses the SCRIPT_NAME variable and appends to the - path and ckan addes the root url causing a duplication of the root path. - - This is a middleware to ensure that even redirects use this logic. - ''' - def __init__(self, app, config): - self.app = app - - def __call__(self, environ, start_response): - # Prevents the variable interfering with the root_path logic - if 'SCRIPT_NAME' in environ: - environ['SCRIPT_NAME'] = '' - - return self.app(environ, start_response) - - -class I18nMiddleware(object): - """I18n Middleware selects the language based on the url - eg /fr/home is French""" - def __init__(self, app, config): - self.app = app - self.default_locale = config.get('ckan.locale_default', 'en') - self.local_list = get_locales_from_config() - - def __call__(self, environ, start_response): - # strip the language selector from the requested url - # and set environ variables for the language selected - # CKAN_LANG is the language code eg en, fr - # CKAN_LANG_IS_DEFAULT is set to True or False - # CKAN_CURRENT_URL is set to the current application url - - # We only update once for a request so we can keep - # the language and original url which helps with 404 pages etc - if 'CKAN_LANG' not in environ: - path_parts = environ['PATH_INFO'].split('/') - if len(path_parts) > 1 and path_parts[1] in self.local_list: - environ['CKAN_LANG'] = path_parts[1] - environ['CKAN_LANG_IS_DEFAULT'] = False - # rewrite url - if len(path_parts) > 2: - environ['PATH_INFO'] = '/'.join([''] + path_parts[2:]) - else: - environ['PATH_INFO'] = '/' - else: - environ['CKAN_LANG'] = self.default_locale - environ['CKAN_LANG_IS_DEFAULT'] = True - - # Current application url - path_info = environ['PATH_INFO'] - # sort out weird encodings - path_info = '/'.join(urllib.quote(pce, '') for pce in path_info.split('/')) - - qs = environ.get('QUERY_STRING') - - if qs: - # sort out weird encodings - qs = urllib.quote(qs, '') - environ['CKAN_CURRENT_URL'] = '%s?%s' % (path_info, qs) - else: - environ['CKAN_CURRENT_URL'] = path_info - - return self.app(environ, start_response) - - -class PageCacheMiddleware(object): - ''' A simple page cache that can store and serve pages. It uses - Redis as storage. It caches pages that have a http status code of - 200, use the GET method. Only non-logged in users receive cached - pages. - Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE - variable.''' - - def __init__(self, app, config): - self.app = app - import redis # only import if used - self.redis = redis # we need to reference this within the class - self.redis_exception = redis.exceptions.ConnectionError - self.redis_connection = None - - def __call__(self, environ, start_response): - - def _start_response(status, response_headers, exc_info=None): - # This wrapper allows us to get the status and headers. - environ['CKAN_PAGE_STATUS'] = status - environ['CKAN_PAGE_HEADERS'] = response_headers - return start_response(status, response_headers, exc_info) - - # Only use cache for GET requests - # REMOTE_USER is used by some tests. - if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'): - return self.app(environ, start_response) - - # If there is a ckan cookie (or auth_tkt) we avoid the cache. - # We want to allow other cookies like google analytics ones :( - cookie_string = environ.get('HTTP_COOKIE') - if cookie_string: - for cookie in cookie_string.split(';'): - if cookie.startswith('ckan') or cookie.startswith('auth_tkt'): - return self.app(environ, start_response) - - # Make our cache key - key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING']) - - # Try to connect if we don't have a connection. Doing this here - # allows the redis server to be unavailable at times. - if self.redis_connection is None: - try: - self.redis_connection = self.redis.StrictRedis() - self.redis_connection.flushdb() - except self.redis_exception: - # Connection may have failed at flush so clear it. - self.redis_connection = None - return self.app(environ, start_response) - - # If cached return cached result - try: - result = self.redis_connection.lrange(key, 0, 2) - except self.redis_exception: - # Connection failed so clear it and return the page as normal. - self.redis_connection = None - return self.app(environ, start_response) - - if result: - headers = json.loads(result[1]) - # Convert headers from list to tuples. - headers = [(str(key), str(value)) for key, value in headers] - start_response(str(result[0]), headers) - # Returning a huge string slows down the server. Therefore we - # cut it up into more usable chunks. - page = result[2] - out = [] - total = len(page) - position = 0 - size = 4096 - while position < total: - out.append(page[position:position + size]) - position += size - return out - - # Generate the response from our application. - page = self.app(environ, _start_response) - - # Only cache http status 200 pages - if not environ['CKAN_PAGE_STATUS'].startswith('200'): - return page - - cachable = False - if environ.get('CKAN_PAGE_CACHABLE'): - cachable = True - - # Cache things if cachable. - if cachable: - # Make sure we consume any file handles etc. - page_string = ''.join(list(page)) - # Use a pipe to add page in a transaction. - pipe = self.redis_connection.pipeline() - pipe.rpush(key, environ['CKAN_PAGE_STATUS']) - pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS'])) - pipe.rpush(key, page_string) - pipe.execute() - return page - - -class TrackingMiddleware(object): - - def __init__(self, app, config): - self.app = app - self.engine = sa.create_engine(config.get('sqlalchemy.url')) - - def __call__(self, environ, start_response): - path = environ['PATH_INFO'] - method = environ.get('REQUEST_METHOD') - if path == '/_tracking' and method == 'POST': - # do the tracking - # get the post data - payload = environ['wsgi.input'].read() - parts = payload.split('&') - data = {} - for part in parts: - k, v = part.split('=') - data[k] = urllib2.unquote(v).decode("utf8") - start_response('200 OK', [('Content-Type', 'text/html')]) - # we want a unique anonomized key for each user so that we do - # not count multiple clicks from the same user. - key = ''.join([ - environ['HTTP_USER_AGENT'], - environ['REMOTE_ADDR'], - environ.get('HTTP_ACCEPT_LANGUAGE', ''), - environ.get('HTTP_ACCEPT_ENCODING', ''), - ]) - key = hashlib.md5(key).hexdigest() - # store key/data here - sql = '''INSERT INTO tracking_raw - (user_key, url, tracking_type) - VALUES (%s, %s, %s)''' - self.engine.execute(sql, key, data.get('url'), data.get('type')) - return [] - return self.app(environ, start_response) - - -def generate_close_and_callback(iterable, callback, environ): - """ - return a generator that passes through items from iterable - then calls callback(environ). - """ - try: - for item in iterable: - yield item - except GeneratorExit: - if hasattr(iterable, 'close'): - iterable.close() - raise - finally: - callback(environ) - - -def execute_on_completion(application, config, callback): - """ - Call callback(environ) once complete response is sent - """ - def inner(environ, start_response): - try: - result = application(environ, start_response) - except: - callback(environ) - raise - return generate_close_and_callback(result, callback, environ) - return inner - - -def cleanup_pylons_response_string(environ): - try: - msg = 'response cleared by pylons response cleanup middleware' - environ['pylons.controller']._py_object.response._body = msg - except (KeyError, AttributeError): - pass diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py new file mode 100644 index 00000000000..33adb1be313 --- /dev/null +++ b/ckan/config/middleware/__init__.py @@ -0,0 +1,132 @@ +# encoding: utf-8 + +"""WSGI app initialization""" + +import webob + +from werkzeug.test import create_environ, run_wsgi_app +from wsgi_party import WSGIParty + +from ckan.config.middleware.flask_app import make_flask_stack +from ckan.config.middleware.pylons_app import make_pylons_stack + +import logging +log = logging.getLogger(__name__) + +# This monkey-patches the webob request object because of the way it messes +# with the WSGI environ. + +# Start of webob.requests.BaseRequest monkey patch +original_charset__set = webob.request.BaseRequest._charset__set + + +def custom_charset__set(self, charset): + original_charset__set(self, charset) + if self.environ.get('CONTENT_TYPE', '').startswith(';'): + self.environ['CONTENT_TYPE'] = '' + +webob.request.BaseRequest._charset__set = custom_charset__set + +webob.request.BaseRequest.charset = property( + webob.request.BaseRequest._charset__get, + custom_charset__set, + webob.request.BaseRequest._charset__del, + webob.request.BaseRequest._charset__get.__doc__) + +# End of webob.requests.BaseRequest monkey patch + + +def make_app(conf, full_stack=True, static_files=True, **app_conf): + ''' + Initialise both the pylons and flask apps, and wrap them in dispatcher + middleware. + ''' + + pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) + flask_app = make_flask_stack(conf) + + app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, + 'flask_app': flask_app}) + + return app + + +class AskAppDispatcherMiddleware(WSGIParty): + + ''' + Establish a 'partyline' to each provided app. Select which app to call + by asking each if they can handle the requested path at PATH_INFO. + + Used to help transition from Pylons to Flask, and should be removed once + Pylons has been deprecated and all app requests are handled by Flask. + + Each app should handle a call to 'can_handle_request(environ)', responding + with a tuple: + (, , []) + where: + `bool` is True if the app can handle the payload url, + `app` is the wsgi app returning the answer + `origin` is an optional string to determine where in the app the url + will be handled, e.g. 'core' or 'extension'. + + Order of precedence if more than one app can handle a url: + Flask Extension > Pylons Extension > Flask Core > Pylons Core + ''' + + def __init__(self, apps=None, invites=(), ignore_missing_services=False): + # Dict of apps managed by this middleware {: , ...} + self.apps = apps or {} + + # A dict of service name => handler mappings. + self.handlers = {} + + # If True, suppress :class:`NoSuchServiceName` errors. Default: False. + self.ignore_missing_services = ignore_missing_services + + self.send_invitations(apps) + + def send_invitations(self, apps): + '''Call each app at the invite route to establish a partyline. Called + on init.''' + PATH = '/__invite__/' + for app_name, app in apps.items(): + environ = create_environ(path=PATH) + environ[self.partyline_key] = self.operator_class(self) + # A reference to the handling app. Used to id the app when + # responding to a handling request. + environ['partyline_handling_app'] = app_name + run_wsgi_app(app, environ) + + def __call__(self, environ, start_response): + '''Determine which app to call by asking each app if it can handle the + url and method defined on the eviron''' + # :::TODO::: Enforce order of precedence for dispatching to apps here. + + app_name = 'pylons_app' # currently defaulting to pylons app + answers = self.ask_around('can_handle_request', environ) + log.debug('Route support answers for {0} {1}: {2}'.format( + environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), + answers)) + available_handlers = [] + for answer in answers: + if len(answer) == 2: + can_handle, asked_app = answer + origin = 'core' + else: + can_handle, asked_app, origin = answer + if can_handle: + available_handlers.append('{0}_{1}'.format(asked_app, origin)) + + # Enforce order of precedence: + # Flask Extension > Pylons Extension > Flask Core > Pylons Core + if available_handlers: + if 'flask_app_extension' in available_handlers: + app_name = 'flask_app' + elif 'pylons_app_extension' in available_handlers: + app_name = 'pylons_app' + elif 'flask_app_core' in available_handlers: + app_name = 'flask_app' + + log.debug('Serving request via {0} app'.format(app_name)) + environ['ckan.app'] = app_name + return self.apps[app_name](environ, start_response) diff --git a/ckan/config/middleware/common_middleware.py b/ckan/config/middleware/common_middleware.py new file mode 100644 index 00000000000..b141b0f3621 --- /dev/null +++ b/ckan/config/middleware/common_middleware.py @@ -0,0 +1,216 @@ +# encoding: utf-8 + +"""Common middleware used by both Flask and Pylons app stacks.""" + +import urllib2 +import hashlib +import urllib +import json + +import sqlalchemy as sa + +from ckan.lib.i18n import get_locales_from_config + + +class I18nMiddleware(object): + """I18n Middleware selects the language based on the url + eg /fr/home is French""" + def __init__(self, app, config): + self.app = app + self.default_locale = config.get('ckan.locale_default', 'en') + self.local_list = get_locales_from_config() + + def __call__(self, environ, start_response): + # strip the language selector from the requested url + # and set environ variables for the language selected + # CKAN_LANG is the language code eg en, fr + # CKAN_LANG_IS_DEFAULT is set to True or False + # CKAN_CURRENT_URL is set to the current application url + + # We only update once for a request so we can keep + # the language and original url which helps with 404 pages etc + if 'CKAN_LANG' not in environ: + path_parts = environ['PATH_INFO'].split('/') + if len(path_parts) > 1 and path_parts[1] in self.local_list: + environ['CKAN_LANG'] = path_parts[1] + environ['CKAN_LANG_IS_DEFAULT'] = False + # rewrite url + if len(path_parts) > 2: + environ['PATH_INFO'] = '/'.join([''] + path_parts[2:]) + else: + environ['PATH_INFO'] = '/' + else: + environ['CKAN_LANG'] = self.default_locale + environ['CKAN_LANG_IS_DEFAULT'] = True + + # Current application url + path_info = environ['PATH_INFO'] + # sort out weird encodings + path_info = \ + '/'.join(urllib.quote(pce, '') for pce in path_info.split('/')) + + qs = environ.get('QUERY_STRING') + + if qs: + # sort out weird encodings + qs = urllib.quote(qs, '') + environ['CKAN_CURRENT_URL'] = '%s?%s' % (path_info, qs) + else: + environ['CKAN_CURRENT_URL'] = path_info + + return self.app(environ, start_response) + + +class RootPathMiddleware(object): + ''' + Prevents the SCRIPT_NAME server variable conflicting with the ckan.root_url + config. The routes package uses the SCRIPT_NAME variable and appends to the + path and ckan addes the root url causing a duplication of the root path. + + This is a middleware to ensure that even redirects use this logic. + ''' + def __init__(self, app, config): + self.app = app + + def __call__(self, environ, start_response): + # Prevents the variable interfering with the root_path logic + if 'SCRIPT_NAME' in environ: + environ['SCRIPT_NAME'] = '' + + return self.app(environ, start_response) + + +class PageCacheMiddleware(object): + ''' A simple page cache that can store and serve pages. It uses + Redis as storage. It caches pages that have a http status code of + 200, use the GET method. Only non-logged in users receive cached + pages. + Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE + variable.''' + + def __init__(self, app, config): + self.app = app + import redis # only import if used + self.redis = redis # we need to reference this within the class + self.redis_exception = redis.exceptions.ConnectionError + self.redis_connection = None + + def __call__(self, environ, start_response): + + def _start_response(status, response_headers, exc_info=None): + # This wrapper allows us to get the status and headers. + environ['CKAN_PAGE_STATUS'] = status + environ['CKAN_PAGE_HEADERS'] = response_headers + return start_response(status, response_headers, exc_info) + + # Only use cache for GET requests + # REMOTE_USER is used by some tests. + if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'): + return self.app(environ, start_response) + + # If there is a ckan cookie (or auth_tkt) we avoid the cache. + # We want to allow other cookies like google analytics ones :( + cookie_string = environ.get('HTTP_COOKIE') + if cookie_string: + for cookie in cookie_string.split(';'): + if cookie.startswith('ckan') or cookie.startswith('auth_tkt'): + return self.app(environ, start_response) + + # Make our cache key + key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING']) + + # Try to connect if we don't have a connection. Doing this here + # allows the redis server to be unavailable at times. + if self.redis_connection is None: + try: + self.redis_connection = self.redis.StrictRedis() + self.redis_connection.flushdb() + except self.redis_exception: + # Connection may have failed at flush so clear it. + self.redis_connection = None + return self.app(environ, start_response) + + # If cached return cached result + try: + result = self.redis_connection.lrange(key, 0, 2) + except self.redis_exception: + # Connection failed so clear it and return the page as normal. + self.redis_connection = None + return self.app(environ, start_response) + + if result: + headers = json.loads(result[1]) + # Convert headers from list to tuples. + headers = [(str(key), str(value)) for key, value in headers] + start_response(str(result[0]), headers) + # Returning a huge string slows down the server. Therefore we + # cut it up into more usable chunks. + page = result[2] + out = [] + total = len(page) + position = 0 + size = 4096 + while position < total: + out.append(page[position:position + size]) + position += size + return out + + # Generate the response from our application. + page = self.app(environ, _start_response) + + # Only cache http status 200 pages + if not environ['CKAN_PAGE_STATUS'].startswith('200'): + return page + + cachable = False + if environ.get('CKAN_PAGE_CACHABLE'): + cachable = True + + # Cache things if cachable. + if cachable: + # Make sure we consume any file handles etc. + page_string = ''.join(list(page)) + # Use a pipe to add page in a transaction. + pipe = self.redis_connection.pipeline() + pipe.rpush(key, environ['CKAN_PAGE_STATUS']) + pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS'])) + pipe.rpush(key, page_string) + pipe.execute() + return page + + +class TrackingMiddleware(object): + + def __init__(self, app, config): + self.app = app + self.engine = sa.create_engine(config.get('sqlalchemy.url')) + + def __call__(self, environ, start_response): + path = environ['PATH_INFO'] + method = environ.get('REQUEST_METHOD') + if path == '/_tracking' and method == 'POST': + # do the tracking + # get the post data + payload = environ['wsgi.input'].read() + parts = payload.split('&') + data = {} + for part in parts: + k, v = part.split('=') + data[k] = urllib2.unquote(v).decode("utf8") + start_response('200 OK', [('Content-Type', 'text/html')]) + # we want a unique anonomized key for each user so that we do + # not count multiple clicks from the same user. + key = ''.join([ + environ['HTTP_USER_AGENT'], + environ['REMOTE_ADDR'], + environ.get('HTTP_ACCEPT_LANGUAGE', ''), + environ.get('HTTP_ACCEPT_ENCODING', ''), + ]) + key = hashlib.md5(key).hexdigest() + # store key/data here + sql = '''INSERT INTO tracking_raw + (user_key, url, tracking_type) + VALUES (%s, %s, %s)''' + self.engine.execute(sql, key, data.get('url'), data.get('type')) + return [] + return self.app(environ, start_response) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py new file mode 100644 index 00000000000..8d37e26d1b8 --- /dev/null +++ b/ckan/config/middleware/flask_app.py @@ -0,0 +1,81 @@ +# encoding: utf-8 + +from flask import Flask +from flask import abort as flask_abort +from flask import request as flask_request +from flask import _request_ctx_stack +from werkzeug.exceptions import HTTPException + +from wsgi_party import WSGIParty, HighAndDry + + +import logging +log = logging.getLogger(__name__) + + +def make_flask_stack(conf): + """ This has to pass the flask app through all the same middleware that + Pylons used """ + + app = CKANFlask(__name__) + + @app.route('/hello', methods=['GET']) + def hello_world(): + return 'Hello World, this is served by Flask' + + @app.route('/hello', methods=['POST']) + def hello_world_post(): + return 'Hello World, this was posted to Flask' + + return app + + +class CKANFlask(Flask): + + '''Extend the Flask class with a special view to join the 'partyline' + established by AskAppDispatcherMiddleware. + + Also provide a 'can_handle_request' method. + ''' + + def __init__(self, import_name, *args, **kwargs): + super(CKANFlask, self).__init__(import_name, *args, **kwargs) + self.add_url_rule('/__invite__/', endpoint='partyline', + view_func=self.join_party) + self.partyline = None + self.partyline_connected = False + self.invitation_context = None + self.app_name = None # A label for the app handling this request + # (this app). + + def join_party(self, request=flask_request): + # Bootstrap, turn the view function into a 404 after registering. + if self.partyline_connected: + # This route does not exist at the HTTP level. + flask_abort(404) + self.invitation_context = _request_ctx_stack.top + self.partyline = request.environ.get(WSGIParty.partyline_key) + self.app_name = request.environ.get('partyline_handling_app') + self.partyline.connect('can_handle_request', self.can_handle_request) + self.partyline_connected = True + return 'ok' + + def can_handle_request(self, environ): + ''' + Decides whether it can handle a request with the Flask app by + matching the request environ against the route mapper + + Returns (True, 'flask_app') if this is the case. + ''' + + # TODO: identify matching urls as core or extension. This will depend + # on how we setup routing in Flask + + urls = self.url_map.bind_to_environ(environ) + try: + endpoint, args = urls.match() + log.debug('Flask route match, endpoint: {0}, args: {1}'.format( + endpoint, args)) + return (True, self.app_name) + except HTTPException: + raise HighAndDry() diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py new file mode 100644 index 00000000000..4d2af590c0c --- /dev/null +++ b/ckan/config/middleware/pylons_app.py @@ -0,0 +1,218 @@ +# encoding: utf-8 + +import os + +from pylons import config +from pylons.wsgiapp import PylonsApp + +from beaker.middleware import CacheMiddleware, SessionMiddleware +from paste.cascade import Cascade +from paste.registry import RegistryManager +from paste.urlparser import StaticURLParser +from paste.deploy.converters import asbool +from pylons.middleware import ErrorHandler, StatusCodeRedirect +from routes.middleware import RoutesMiddleware +from repoze.who.config import WhoConfig +from repoze.who.middleware import PluggableAuthenticationMiddleware +from fanstatic import Fanstatic + +from ckan.plugins import PluginImplementations +from ckan.plugins.interfaces import IMiddleware +import ckan.lib.uploader as uploader +from ckan.config.environment import load_environment +import ckan.lib.app_globals as app_globals +from ckan.config.middleware import common_middleware + +import logging +log = logging.getLogger(__name__) + + +def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): + """Create a Pylons WSGI application and return it + + ``conf`` + The inherited configuration for this application. Normally from + the [DEFAULT] section of the Paste ini file. + + ``full_stack`` + Whether this application provides a full WSGI stack (by default, + meaning it handles its own exceptions and errors). Disable + full_stack when this application is "managed" by another WSGI + middleware. + + ``static_files`` + Whether this application serves its own static files; disable + when another web server is responsible for serving them. + + ``app_conf`` + The application's local configuration. Normally specified in + the [app:] section of the Paste ini file (where + defaults to main). + + """ + # Configure the Pylons environment + load_environment(conf, app_conf) + + # The Pylons WSGI app + app = PylonsApp() + # set pylons globals + app_globals.reset() + + for plugin in PluginImplementations(IMiddleware): + app = plugin.make_middleware(app, config) + + # Routing/Session/Cache Middleware + app = RoutesMiddleware(app, config['routes.map']) + # we want to be able to retrieve the routes middleware to be able to update + # the mapper. We store it in the pylons config to allow this. + config['routes.middleware'] = app + app = SessionMiddleware(app, config) + app = CacheMiddleware(app, config) + + # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) + # app = QueueLogMiddleware(app) + if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): + app = execute_on_completion(app, config, + cleanup_pylons_response_string) + + # Fanstatic + if asbool(config.get('debug', False)): + fanstatic_config = { + 'versioning': True, + 'recompute_hashes': True, + 'minified': False, + 'bottom': True, + 'bundle': False, + } + else: + fanstatic_config = { + 'versioning': True, + 'recompute_hashes': False, + 'minified': True, + 'bottom': True, + 'bundle': True, + } + app = Fanstatic(app, **fanstatic_config) + + for plugin in PluginImplementations(IMiddleware): + try: + app = plugin.make_error_log_middleware(app, config) + except AttributeError: + log.critical('Middleware class {0} is missing the method' + 'make_error_log_middleware.' + .format(plugin.__class__.__name__)) + + if asbool(full_stack): + # Handle Python exceptions + app = ErrorHandler(app, conf, **config['pylons.errorware']) + + # Display error documents for 400, 403, 404 status codes (and + # 500 when debug is disabled) + if asbool(config['debug']): + app = StatusCodeRedirect(app, [400, 403, 404]) + else: + app = StatusCodeRedirect(app, [400, 403, 404, 500]) + + # Initialize repoze.who + who_parser = WhoConfig(conf['here']) + who_parser.parse(open(app_conf['who.config_file'])) + + app = PluggableAuthenticationMiddleware( + app, + who_parser.identifiers, + who_parser.authenticators, + who_parser.challengers, + who_parser.mdproviders, + who_parser.request_classifier, + who_parser.challenge_decider, + logging.getLogger('repoze.who'), + logging.WARN, # ignored + who_parser.remote_user_key + ) + + # Establish the Registry for this application + app = RegistryManager(app) + + app = common_middleware.I18nMiddleware(app, config) + + if asbool(static_files): + # Serve static files + static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ + else int(config.get('ckan.static_max_age', 3600)) + + static_app = StaticURLParser(config['pylons.paths']['static_files'], + cache_max_age=static_max_age) + static_parsers = [static_app, app] + + storage_directory = uploader.get_storage_path() + if storage_directory: + path = os.path.join(storage_directory, 'storage') + try: + os.makedirs(path) + except OSError, e: + # errno 17 is file already exists + if e.errno != 17: + raise + + storage_app = StaticURLParser(path, cache_max_age=static_max_age) + static_parsers.insert(0, storage_app) + + # Configurable extra static file paths + extra_static_parsers = [] + for public_path in config.get('extra_public_paths', '').split(','): + if public_path.strip(): + extra_static_parsers.append( + StaticURLParser(public_path.strip(), + cache_max_age=static_max_age) + ) + app = Cascade(extra_static_parsers + static_parsers) + + # Page cache + if asbool(config.get('ckan.page_cache_enabled')): + app = common_middleware.PageCacheMiddleware(app, config) + + # Tracking + if asbool(config.get('ckan.tracking_enabled', 'false')): + app = common_middleware.TrackingMiddleware(app, config) + + app = common_middleware.RootPathMiddleware(app, config) + + return app + + +def generate_close_and_callback(iterable, callback, environ): + """ + return a generator that passes through items from iterable + then calls callback(environ). + """ + try: + for item in iterable: + yield item + except GeneratorExit: + if hasattr(iterable, 'close'): + iterable.close() + raise + finally: + callback(environ) + + +def execute_on_completion(application, config, callback): + """ + Call callback(environ) once complete response is sent + """ + def inner(environ, start_response): + try: + result = application(environ, start_response) + except: + callback(environ) + raise + return generate_close_and_callback(result, callback, environ) + return inner + + +def cleanup_pylons_response_string(environ): + try: + msg = 'response cleared by pylons response cleanup middleware' + environ['pylons.controller']._py_object.response._body = msg + except (KeyError, AttributeError): + pass diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 623341185b7..45478e322b9 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -8,7 +8,8 @@ import ckan.plugins as p import ckan.tests.helpers as helpers -from ckan.config.middleware import AskAppDispatcherMiddleware, CKANFlask +from ckan.config.middleware import AskAppDispatcherMiddleware +from ckan.config.middleware.flask_app import CKANFlask from ckan.controllers.partyline import PartylineController From 2a726ab6045750906cb4061a96431905b03357ad Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 16 Jun 2016 13:55:34 +0100 Subject: [PATCH 058/210] [#3117] Remove site_url_nice from app_globals --- ckan/lib/app_globals.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index 9f0b32d6659..e62ec879537 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -169,13 +169,9 @@ def get_config_value(key, default=''): for key in schema.keys(): get_config_value(key) - # cusom styling + # custom styling main_css = get_config_value('ckan.main_css', '/base/css/main.css') set_main_css(main_css) - # site_url_nice - site_url_nice = app_globals.site_url.replace('http://', '') - site_url_nice = site_url_nice.replace('www.', '') - app_globals.site_url_nice = site_url_nice if app_globals.site_logo: app_globals.header_class = 'header-image' From 8866961fe236ecdd8b2ed6eaf7510b6bd6f3daf3 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 17 Jun 2016 12:21:51 +0100 Subject: [PATCH 059/210] [#3120] Move _get_page_number to helpers --- ckan/controllers/feed.py | 4 ++-- ckan/controllers/group.py | 4 ++-- ckan/controllers/package.py | 2 +- ckan/controllers/revision.py | 2 +- ckan/controllers/tag.py | 2 +- ckan/controllers/user.py | 2 +- ckan/lib/base.py | 18 ------------------ ckan/lib/helpers.py | 21 +++++++++++++++++++++ 8 files changed, 29 insertions(+), 26 deletions(-) diff --git a/ckan/controllers/feed.py b/ckan/controllers/feed.py index 3b41af9d80d..9da30215a4b 100644 --- a/ckan/controllers/feed.py +++ b/ckan/controllers/feed.py @@ -310,7 +310,7 @@ def custom(self): search_params[param] = value fq += ' %s:"%s"' % (param, value) - page = self._get_page_number(request.params) + page = h.get_page_number(request.params) limit = ITEMS_LIMIT data_dict = { @@ -455,7 +455,7 @@ def _parse_url_params(self): Returns the constructed search-query dict, and the valid URL query parameters. """ - page = self._get_page_number(request.params) + page = h.get_page_number(request.params) limit = ITEMS_LIMIT data_dict = { diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index bd353537aaa..fb05af7a05f 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -150,7 +150,7 @@ def add_group_type(cls, group_type): def index(self): group_type = self._guess_group_type() - page = self._get_page_number(request.params) or 1 + page = h.get_page_number(request.params) or 1 items_per_page = 21 context = {'model': model, 'session': model.Session, @@ -247,7 +247,7 @@ def _read(self, id, limit, group_type): context['return_query'] = True - page = self._get_page_number(request.params) + page = h.get_page_number(request.params) # most search operations should reset the page counter: params_nopage = [(k, v) for k, v in request.params.items() diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 5b41aa95e81..3aed9543437 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -146,7 +146,7 @@ def search(self): # unicode format (decoded from utf8) q = c.q = request.params.get('q', u'') c.query_error = False - page = self._get_page_number(request.params) + page = h.get_page_number(request.params) limit = g.datasets_per_page diff --git a/ckan/controllers/revision.py b/ckan/controllers/revision.py index 2c5962f943b..dd3527cc4ea 100644 --- a/ckan/controllers/revision.py +++ b/ckan/controllers/revision.py @@ -124,7 +124,7 @@ def list(self): query = model.Session.query(model.Revision) c.page = h.Page( collection=query, - page=self._get_page_number(request.params), + page=h.get_page_number(request.params), url=h.pager_url, items_per_page=20 ) diff --git a/ckan/controllers/tag.py b/ckan/controllers/tag.py index 67b761af444..dc857d3b7d6 100644 --- a/ckan/controllers/tag.py +++ b/ckan/controllers/tag.py @@ -36,7 +36,7 @@ def index(self): data_dict = {'all_fields': True} if c.q: - page = self._get_page_number(request.params) + page = h.get_page_number(request.params) data_dict['q'] = c.q data_dict['limit'] = LIMIT data_dict['offset'] = (page - 1) * LIMIT diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index c7637fa122d..3f758f7bf67 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -97,7 +97,7 @@ def _get_repoze_handler(self, handler_name): handler_name) def index(self): - page = self._get_page_number(request.params) + page = h.get_page_number(request.params) c.q = request.params.get('q', '') c.order_by = request.params.get('order_by', 'name') diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 86b10c1399c..1dfe6476ab1 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -377,24 +377,6 @@ def _get_user_for_apikey(self): user = query.filter_by(apikey=apikey).first() return user - def _get_page_number(self, params, key='page', default=1): - """ - Returns the page number from the provided params after - verifies that it is an integer. - - If it fails it will abort the request with a 400 error - """ - p = params.get(key, default) - - try: - p = int(p) - if p < 1: - raise ValueError("Negative number not allowed") - except ValueError, e: - abort(400, ('"page" parameter must be a positive integer')) - - return p - # Include the '_' function in the public names __all__ = [__name for __name in locals().keys() if not __name.startswith('_') diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 5ae9929416a..b1c061317b5 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1081,6 +1081,27 @@ def _range(self, regexp_match): return re.sub(current_page_span, current_page_link, html) +@core_helper +def get_page_number(params, key='page', default=1): + ''' + Return the page number from the provided params after verifying that it is + an positive integer. + + If it fails it will abort the request with a 400 error. + ''' + p = params.get(key, default) + + try: + p = int(p) + if p < 1: + raise ValueError("Negative number not allowed") + except ValueError: + import ckan.lib.base as base + base.abort(400, ('"page" parameter must be a positive integer')) + + return p + + @core_helper def get_display_timezone(): ''' Returns a pytz timezone for the display_timezone setting in the From 0d9f680fbdaa2abfc758b974d4c30ea67299ed55 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 17 Jun 2016 12:58:56 +0100 Subject: [PATCH 060/210] [#3120] Remove module var PAGINATE_ITEMS_PER_PAGE The PAGINATE_ITEMS_PER_PAGE variable was used in previous versions of the BaseController, but the related code has since been removed. --- ckan/lib/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 1dfe6476ab1..959053f0e8d 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -34,8 +34,6 @@ log = logging.getLogger(__name__) -PAGINATE_ITEMS_PER_PAGE = 50 - APIKEY_HEADER_NAME_KEY = 'apikey_header_name' APIKEY_HEADER_NAME_DEFAULT = 'X-CKAN-API-Key' From 811d8889bf6a49d5eb2f1a833fa26a0d45eec4f2 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 17 Jun 2016 13:04:30 +0100 Subject: [PATCH 061/210] [#3120] Remove module var ALLOWED_FIELDSET_PARAMS. ALLOWED_FIELDSET_PARAMS was used in the BaseController, but the method that used it has since been removed. --- ckan/lib/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 959053f0e8d..5b0241a4cef 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -37,8 +37,6 @@ APIKEY_HEADER_NAME_KEY = 'apikey_header_name' APIKEY_HEADER_NAME_DEFAULT = 'X-CKAN-API-Key' -ALLOWED_FIELDSET_PARAMS = ['package_form', 'restrict'] - def abort(status_code=None, detail='', headers=None, comment=None): '''Abort the current request immediately by returning an HTTP exception. From 3d0f8abd511d5f07f9a28c7c478a801cf5fe19fd Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 17 Jun 2016 14:25:35 +0100 Subject: [PATCH 062/210] [#3121] Change test to prevent deprecation warning --- ckan/tests/controllers/test_user.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/tests/controllers/test_user.py b/ckan/tests/controllers/test_user.py index cedeca245ce..b3f63d12bc7 100644 --- a/ckan/tests/controllers/test_user.py +++ b/ckan/tests/controllers/test_user.py @@ -174,7 +174,9 @@ def test_user_logout_url_redirect(self): @helpers.change_config('ckan.root_path', '/my/prefix') def test_non_root_user_logout_url_redirect(self): - '''_logout url redirects to logged out page. + ''' + _logout url redirects to logged out page with `ckan.root_path` + prefixed. Note: this doesn't test the actual logout of a logged in user, just the associated redirect. @@ -183,10 +185,8 @@ def test_non_root_user_logout_url_redirect(self): logout_url = url_for(controller='user', action='logout') logout_response = app.get(logout_url, status=302) - try: - final_response = helpers.webtest_maybe_follow(logout_response) - except Exception as e: - assert_true('/my/prefix/user/logout' in e.message) + assert_equal(logout_response.status_int, 302) + assert_true('/my/prefix/user/logout' in logout_response.location) class TestUser(helpers.FunctionalTestBase): From 5ce8284c1f127f1f8b4279bbc96a2422b13bef6a Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 17 Jun 2016 14:40:32 +0100 Subject: [PATCH 063/210] [#3120] Change working of abort message. 'key' can be defined in the call, so may not always be 'page'. --- ckan/lib/helpers.py | 2 +- ckan/tests/controllers/test_feed.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index b1c061317b5..a19ade3132c 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1097,7 +1097,7 @@ def get_page_number(params, key='page', default=1): raise ValueError("Negative number not allowed") except ValueError: import ckan.lib.base as base - base.abort(400, ('"page" parameter must be a positive integer')) + base.abort(400, ('"{key}" parameter must be a positive integer')) return p diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index 0fc448544eb..57adcade8ea 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -2,7 +2,6 @@ from routes import url_for -from ckan import model import ckan.tests.helpers as helpers import ckan.tests.factories as factories @@ -15,7 +14,7 @@ def test_atom_feed_page_zero_gives_error(self): id=group['name']) + '?page=0' app = self._get_test_app() res = app.get(offset, status=400) - assert '"page" parameter must be a positive integer' in res, res + assert '"{key}" parameter must be a positive integer' in res, res def test_atom_feed_page_negative_gives_error(self): group = factories.Group() @@ -23,7 +22,7 @@ def test_atom_feed_page_negative_gives_error(self): id=group['name']) + '?page=-2' app = self._get_test_app() res = app.get(offset, status=400) - assert '"page" parameter must be a positive integer' in res, res + assert '"{key}" parameter must be a positive integer' in res, res def test_atom_feed_page_not_int_gives_error(self): group = factories.Group() @@ -31,4 +30,4 @@ def test_atom_feed_page_not_int_gives_error(self): id=group['name']) + '?page=abc' app = self._get_test_app() res = app.get(offset, status=400) - assert '"page" parameter must be a positive integer' in res, res + assert '"{key}" parameter must be a positive integer' in res, res From 91bf17e269269973f967fb84e62cd36b7bccf040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 16:42:55 +0200 Subject: [PATCH 064/210] remove duplicated import --- ckan/tests/logic/test_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/tests/logic/test_validators.py b/ckan/tests/logic/test_validators.py index 8a5d854e1ce..9e8e1090259 100644 --- a/ckan/tests/logic/test_validators.py +++ b/ckan/tests/logic/test_validators.py @@ -11,13 +11,13 @@ import mock import nose.tools -import ckan.tests.factories as factories # Import some test helper functions from another module. # This is bad (test modules shouldn't share code with eachother) but because of # the way validator functions are organised in CKAN (in different modules in # different places in the code) we have to either do this or introduce a shared # test helper functions module (which we also don't want to do). import ckan.tests.lib.navl.test_validators as t + import ckan.lib.navl.dictization_functions as df import ckan.logic.validators as validators import ckan.tests.factories as factories From 282543be441329b66d2532c3c4dccdc981dd09f1 Mon Sep 17 00:00:00 2001 From: Cody Date: Fri, 17 Jun 2016 14:44:01 +0000 Subject: [PATCH 065/210] Use dict comprehension instead of dict([...]) --- ckan/lib/config_tool.py | 5 ++--- ckan/lib/create_test_data.py | 2 +- ckan/lib/dictization/model_save.py | 6 ++---- ckan/logic/schema.py | 2 +- ckan/model/package.py | 2 +- 5 files changed, 7 insertions(+), 10 deletions(-) diff --git a/ckan/lib/config_tool.py b/ckan/lib/config_tool.py index d5118c9d436..9ca8dfa42c6 100644 --- a/ckan/lib/config_tool.py +++ b/ckan/lib/config_tool.py @@ -189,9 +189,8 @@ def insert_new_sections(new_sections): # at start of new section, write the 'add'ed options for option in changes.get(section, 'add'): write_option(option) - options_to_edit_in_this_section = dict( - [(option.key, option) - for option in changes.get(section, 'edit')]) + options_to_edit_in_this_section = {option.key: option + for option in changes.get(section, 'edit')} continue existing_option = parse_option_string(section, line) if not existing_option: diff --git a/ckan/lib/create_test_data.py b/ckan/lib/create_test_data.py index 0dc4df0bda7..73ee6633053 100644 --- a/ckan/lib/create_test_data.py +++ b/ckan/lib/create_test_data.py @@ -187,7 +187,7 @@ def create_arbitrary(cls, package_dicts, relationships=[], if not isinstance(v, datetime.datetime): v = unicode(v) non_extras[str(k)] = v - extras = dict([(str(k), unicode(v)) for k, v in res_dict.get('extras', {}).items()]) + extras = {str(k): unicode(v) for k, v in res_dict.get('extras', {}).items()} pkg.add_resource(extras=extras, **non_extras) elif attr == 'tags': if isinstance(val, (str, unicode)): diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index 9420b86f5f1..60476666682 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -157,10 +157,8 @@ def package_tag_list_save(tag_dicts, package, context): for package_tag in package.package_tag_all) - tag_package_tag_inactive = dict( - [ (tag,pt) for tag,pt in tag_package_tag.items() if - pt.state in ['deleted'] ] - ) + tag_package_tag_inactive = {tag: pt for tag,pt in tag_package_tag.items() if + pt.state in ['deleted']} tag_name_vocab = set() tags = set() diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index fa2cc484979..f6b369a26f3 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -618,7 +618,7 @@ def create_schema_for_required_keys(keys): ''' helper function that creates a schema definition where each key from keys is validated against ``not_missing``. ''' - schema = dict([(x, [not_missing]) for x in keys]) + schema = {x: [not_missing] for x in keys} return schema diff --git a/ckan/model/package.py b/ckan/model/package.py index 60eab5fadef..6526dc602d8 100644 --- a/ckan/model/package.py +++ b/ckan/model/package.py @@ -204,7 +204,7 @@ def as_dict(self, ref_package_by='name', ref_group_by='name'): groups = [getattr(group, ref_group_by) for group in self.get_groups()] groups.sort() _dict['groups'] = groups - _dict['extras'] = dict([(key, value) for key, value in self.extras.items()]) + _dict['extras'] = {key: value for key, value in self.extras.items()} _dict['ratings_average'] = self.get_average_rating() _dict['ratings_count'] = len(self.ratings) _dict['resources'] = [res.as_dict(core_columns_only=False) \ From 550385128cf058a9aa863c1e17924fc4a9b3d395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 17:15:34 +0200 Subject: [PATCH 066/210] [#3123] fix indentation for pep8 --- ckan/lib/config_tool.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/lib/config_tool.py b/ckan/lib/config_tool.py index 9ca8dfa42c6..54060b77671 100644 --- a/ckan/lib/config_tool.py +++ b/ckan/lib/config_tool.py @@ -190,7 +190,8 @@ def insert_new_sections(new_sections): for option in changes.get(section, 'add'): write_option(option) options_to_edit_in_this_section = {option.key: option - for option in changes.get(section, 'edit')} + for option + in changes.get(section, 'edit')} continue existing_option = parse_option_string(section, line) if not existing_option: From 82fe4fd9bbe50c7e8285968f2caeb4688fc19f66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 17:43:16 +0200 Subject: [PATCH 067/210] [#3126] remove backported `subprocess.check_output` --- ckan/lib/util.py | 47 ----------------------------- ckan/tests/test_coding_standards.py | 10 +++--- doc/conf.py | 4 +-- 3 files changed, 7 insertions(+), 54 deletions(-) delete mode 100644 ckan/lib/util.py diff --git a/ckan/lib/util.py b/ckan/lib/util.py deleted file mode 100644 index 3cf19c348d1..00000000000 --- a/ckan/lib/util.py +++ /dev/null @@ -1,47 +0,0 @@ -# encoding: utf-8 - -'''Shared utility functions for any Python code to use. - -Unlike :py:mod:`ckan.lib.helpers`, the functions in this module are not -available to templates. - -''' -import subprocess - - -# We implement our own check_output() function because -# subprocess.check_output() isn't in Python 2.6. -# This code is copy-pasted from Python 2.7 and adapted to make it work with -# Python 2.6. -# http://hg.python.org/cpython/file/d37f963394aa/Lib/subprocess.py#l544 -def check_output(*popenargs, **kwargs): - r"""Run command with arguments and return its output as a byte string. - - If the exit code was non-zero it raises a CalledProcessError. The - CalledProcessError object will have the return code in the returncode - attribute and output in the output attribute. - - The arguments are the same as for the Popen constructor. Example: - - >>> check_output(["ls", "-l", "/dev/null"]) - 'crw-rw-rw- 1 root root 1, 3 Oct 18 2007 /dev/null\n' - - The stdout argument is not allowed as it is used internally. - To capture standard error in the result, use stderr=STDOUT. - - >>> check_output(["/bin/sh", "-c", - ... "ls -l non_existent_file ; exit 0"], - ... stderr=STDOUT) - 'ls: non_existent_file: No such file or directory\n' - """ - if 'stdout' in kwargs: - raise ValueError('stdout argument not allowed, it will be overridden.') - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd) - return output diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index 8726f277bd3..6010449676d 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -15,8 +15,6 @@ import re import subprocess -import ckan.lib.util as util - def test_building_the_docs(): '''There should be no warnings or errors when building the Sphinx docs. @@ -28,8 +26,12 @@ def test_building_the_docs(): ''' try: - output = util.check_output( - ['python', 'setup.py', 'build_sphinx', '--all-files', '--fresh-env'], + output = subprocess.check_output( + ['python', + 'setup.py', + 'build_sphinx', + '--all-files', + '--fresh-env'], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: assert False, ( diff --git a/doc/conf.py b/doc/conf.py index 635dba13f16..3ed57700fbb 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -18,8 +18,6 @@ import os import subprocess -import ckan.lib.util as util - # If your extensions (or modules documented by autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. @@ -126,7 +124,7 @@ def latest_release_tag(): This requires git to be installed. ''' - git_tags = util.check_output( + git_tags = subprocess.check_output( ['git', 'tag', '-l'], stderr=subprocess.STDOUT).split() # FIXME: We could do more careful pattern matching against ckan-X.Y.Z here. From 1efb47a879877d88250cf1016be2bda48c1ff54c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 17:53:28 +0200 Subject: [PATCH 068/210] [#3126] don't check if methodcaller is available - `operator.methodcaller` was implemented in python2.7. Now that ckan only supports 2.7 it is no longer required to check if the import is possible or not. We can besure that it is there. --- ckan/model/extension.py | 12 ++---------- ckan/tests/legacy/test_coding_standards.py | 1 - 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/ckan/model/extension.py b/ckan/model/extension.py index b471534a45a..64c68c8f1fa 100644 --- a/ckan/model/extension.py +++ b/ckan/model/extension.py @@ -4,23 +4,17 @@ Provides bridges between the model and plugin PluginImplementationss """ import logging +from operator import methodcaller from sqlalchemy.orm.interfaces import MapperExtension from sqlalchemy.orm.session import SessionExtension import ckan.plugins as plugins -try: - from operator import methodcaller -except ImportError: - def methodcaller(name, *args, **kwargs): - "Replaces stdlib operator.methodcaller in python <2.6" - def caller(obj): - return getattr(obj, name)(*args, **kwargs) - return caller log = logging.getLogger(__name__) + class ObserverNotifier(object): """ Mixin for hooking into SQLAlchemy @@ -93,7 +87,6 @@ def notify_observers(self, func): for observer in plugins.PluginImplementations(plugins.ISession): func(observer) - def after_begin(self, session, transaction, connection): return self.notify_observers( methodcaller('after_begin', session, transaction, connection) @@ -123,4 +116,3 @@ def after_rollback(self, session): return self.notify_observers( methodcaller('after_rollback', session) ) - diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py index b3c556833d5..b9a5f0d3142 100644 --- a/ckan/tests/legacy/test_coding_standards.py +++ b/ckan/tests/legacy/test_coding_standards.py @@ -488,7 +488,6 @@ class TestPep8(object): 'ckan/model/authz.py', 'ckan/model/dashboard.py', 'ckan/model/domain_object.py', - 'ckan/model/extension.py', 'ckan/model/follower.py', 'ckan/model/group.py', 'ckan/model/group_extra.py', From f8bd2cd72abfea9911892543309269d59d2925c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 18:22:45 +0200 Subject: [PATCH 069/210] [#3126] remove assert_(not)_in from helpers - `assert_in` and `assert_not_in` were duplicated in `ckan/tests/legacy/__init__.py` and `ckan.tests.helpers` because they were not available in python2.6 - This commit removes the back porting from those files and just imports from nose.tools directly everywhere --- ckan/tests/controllers/test_api.py | 3 +-- ckan/tests/controllers/test_group.py | 3 +-- ckan/tests/controllers/test_organization.py | 4 ++-- ckan/tests/controllers/test_package.py | 3 +-- ckan/tests/controllers/test_tags.py | 3 +-- ckan/tests/helpers.py | 12 +----------- ckan/tests/legacy/__init__.py | 9 --------- ckan/tests/legacy/functional/api/test_api.py | 4 ++-- ckan/tests/legacy/lib/test_dictization.py | 4 ++-- ckan/tests/legacy/models/test_group.py | 5 ++++- ckan/tests/lib/search/test_index.py | 6 +----- ckan/tests/lib/test_mailer.py | 4 +--- ckanext/example_igroupform/tests/test_controllers.py | 3 +-- ckanext/example_iuploader/test/test_plugin.py | 2 +- ckanext/example_theme/custom_emails/tests.py | 5 +---- 15 files changed, 20 insertions(+), 50 deletions(-) diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 9cab9d2c1dd..5050ddb32aa 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -7,10 +7,9 @@ import json from routes import url_for -from nose.tools import assert_equal +from nose.tools import assert_equal, assert_in import ckan.tests.helpers as helpers -from ckan.tests.helpers import assert_in from ckan.tests import factories from ckan import model diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py index bae63cafc45..67e06c92ce9 100644 --- a/ckan/tests/controllers/test_group.py +++ b/ckan/tests/controllers/test_group.py @@ -1,7 +1,7 @@ # encoding: utf-8 from bs4 import BeautifulSoup -from nose.tools import assert_equal, assert_true +from nose.tools import assert_equal, assert_true, assert_in from routes import url_for @@ -9,7 +9,6 @@ import ckan.model as model from ckan.tests import factories -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckan/tests/controllers/test_organization.py b/ckan/tests/controllers/test_organization.py index b8e038bcf1e..ed8da4a48a8 100644 --- a/ckan/tests/controllers/test_organization.py +++ b/ckan/tests/controllers/test_organization.py @@ -1,12 +1,12 @@ # encoding: utf-8 from bs4 import BeautifulSoup -from nose.tools import assert_equal, assert_true +from nose.tools import assert_equal, assert_true, assert_in from routes import url_for from mock import patch from ckan.tests import factories, helpers -from ckan.tests.helpers import webtest_submit, submit_and_follow, assert_in +from ckan.tests.helpers import webtest_submit, submit_and_follow class TestOrganizationNew(helpers.FunctionalTestBase): diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index aac6ba8b864..853414ed4ef 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -6,6 +6,7 @@ assert_not_equal, assert_raises, assert_true, + assert_in ) from mock import patch, MagicMock @@ -17,10 +18,8 @@ import ckan.tests.helpers as helpers import ckan.tests.factories as factories -from ckan.tests.helpers import assert_in -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckan/tests/controllers/test_tags.py b/ckan/tests/controllers/test_tags.py index d8c17933515..77d7e083311 100644 --- a/ckan/tests/controllers/test_tags.py +++ b/ckan/tests/controllers/test_tags.py @@ -3,7 +3,7 @@ import math import string -from nose.tools import assert_equal, assert_true, assert_false +from nose.tools import assert_equal, assert_true, assert_false, assert_in from bs4 import BeautifulSoup from routes import url_for @@ -11,7 +11,6 @@ import ckan.tests.helpers as helpers from ckan.tests import factories -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index e43093076f2..7f8a0ff77c5 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -22,6 +22,7 @@ import webtest from pylons import config import nose.tools +from nose.tools import assert_in, assert_not_in import mock import ckan.lib.search as search @@ -30,17 +31,6 @@ import ckan.logic as logic -try: - from nose.tools import assert_in, assert_not_in -except ImportError: - # Python 2.6 doesn't have these, so define them here - def assert_in(a, b, msg=None): - assert a in b, msg or '%r was not in %r' % (a, b) - - def assert_not_in(a, b, msg=None): - assert a not in b, msg or '%r was in %r' % (a, b) - - def reset_db(): '''Reset CKAN's database. diff --git a/ckan/tests/legacy/__init__.py b/ckan/tests/legacy/__init__.py index acaafc60b52..fcbf89c7134 100644 --- a/ckan/tests/legacy/__init__.py +++ b/ckan/tests/legacy/__init__.py @@ -355,15 +355,6 @@ def skip_test(*args): def clear_flash(res=None): messages = h._flash.pop_messages() -try: - from nose.tools import assert_in, assert_not_in -except ImportError: - def assert_in(a, b, msg=None): - assert a in b, msg or '%r was not in %r' % (a, b) - def assert_not_in(a, b, msg=None): - assert a not in b, msg or '%r was in %r' % (a, b) - - class StatusCodes: STATUS_200_OK = 200 STATUS_201_CREATED = 201 diff --git a/ckan/tests/legacy/functional/api/test_api.py b/ckan/tests/legacy/functional/api/test_api.py index 7c7a1756008..7f172ed1129 100644 --- a/ckan/tests/legacy/functional/api/test_api.py +++ b/ckan/tests/legacy/functional/api/test_api.py @@ -2,10 +2,10 @@ import json +from nose.tools import assert_in + from ckan.tests.legacy.functional.api.base import * -import ckan.tests.legacy -assert_in = ckan.tests.legacy.assert_in class ApiTestCase(ApiTestCase, ControllerTestCase): diff --git a/ckan/tests/legacy/lib/test_dictization.py b/ckan/tests/legacy/lib/test_dictization.py index 864a22b06c0..6a7cfbf8399 100644 --- a/ckan/tests/legacy/lib/test_dictization.py +++ b/ckan/tests/legacy/lib/test_dictization.py @@ -1,10 +1,10 @@ # encoding: utf-8 -from ckan.tests.legacy import assert_equal, assert_not_in, assert_in +from nose.tools import assert_equal, assert_not_in, assert_in from pprint import pprint, pformat from difflib import unified_diff -import ckan.lib.search as search +import ckan.lib.search as search from ckan.lib.create_test_data import CreateTestData from ckan import model from ckan.lib.dictization import (table_dictize, diff --git a/ckan/tests/legacy/models/test_group.py b/ckan/tests/legacy/models/test_group.py index 9d0e76acd96..a4da7148088 100644 --- a/ckan/tests/legacy/models/test_group.py +++ b/ckan/tests/legacy/models/test_group.py @@ -1,9 +1,12 @@ # encoding: utf-8 -from ckan.tests.legacy import assert_equal, assert_in, assert_not_in, CreateTestData +from nose.tools import assert_in, assert_not_in, assert_equal + +from ckan.tests.legacy import CreateTestData import ckan.model as model + class TestGroup(object): @classmethod diff --git a/ckan/tests/lib/search/test_index.py b/ckan/tests/lib/search/test_index.py index dd783e6d500..dbee706d983 100644 --- a/ckan/tests/lib/search/test_index.py +++ b/ckan/tests/lib/search/test_index.py @@ -3,17 +3,13 @@ import datetime import hashlib import json -import nose.tools import nose +from nose.tools import assert_equal, assert_in, assert_not_in from pylons import config import ckan.lib.search as search import ckan.tests.helpers as helpers -assert_equal = nose.tools.assert_equal -assert_in = helpers.assert_in -assert_not_in = helpers.assert_not_in - class TestSearchIndex(object): diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index b45b41b6e47..143be8373b4 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from nose.tools import assert_equal, assert_raises +from nose.tools import assert_equal, assert_raises, assert_in from pylons import config from email.mime.text import MIMEText from email.parser import Parser @@ -16,8 +16,6 @@ import ckan.tests.helpers as helpers import ckan.tests.factories as factories -assert_in = helpers.assert_in - class MailerBase(SmtpServerHarness): diff --git a/ckanext/example_igroupform/tests/test_controllers.py b/ckanext/example_igroupform/tests/test_controllers.py index b6f03eaf7e6..be598a13aae 100644 --- a/ckanext/example_igroupform/tests/test_controllers.py +++ b/ckanext/example_igroupform/tests/test_controllers.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from nose.tools import assert_equal +from nose.tools import assert_equal, assert_in from routes import url_for import ckan.plugins as plugins @@ -8,7 +8,6 @@ import ckan.model as model from ckan.tests import factories -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckanext/example_iuploader/test/test_plugin.py b/ckanext/example_iuploader/test/test_plugin.py index f99dc6b15f9..08daf634d21 100644 --- a/ckanext/example_iuploader/test/test_plugin.py +++ b/ckanext/example_iuploader/test/test_plugin.py @@ -6,6 +6,7 @@ from mock import patch from nose.tools import ( assert_equal, + assert_in, assert_is_instance ) from pyfakefs import fake_filesystem @@ -19,7 +20,6 @@ import ckan.tests.helpers as helpers import ckanext.example_iuploader.plugin as plugin -assert_in = helpers.assert_in webtest_submit = helpers.webtest_submit submit_and_follow = helpers.submit_and_follow diff --git a/ckanext/example_theme/custom_emails/tests.py b/ckanext/example_theme/custom_emails/tests.py index 3aaf30e40f4..5c72852d15a 100644 --- a/ckanext/example_theme/custom_emails/tests.py +++ b/ckanext/example_theme/custom_emails/tests.py @@ -11,10 +11,7 @@ from ckan.tests.lib.test_mailer import MailerBase import ckan.tests.helpers as helpers -from nose.tools import assert_equal - - -assert_in = helpers.assert_in +from nose.tools import assert_equal, assert_in class TestExampleCustomEmailsPlugin(MailerBase): From 7ec19e36c7c9049a650ba88f8f4bd310ae4e5fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Fri, 17 Jun 2016 18:34:32 +0200 Subject: [PATCH 070/210] [#3126] remove check for skipping tests - removes some code that was used to skip tests and change solr settings under python2.6 which is not tested against anymore --- ckanext/datapusher/tests/test.py | 17 ----------------- ckanext/datastore/tests/test_create.py | 6 ------ ckanext/resourceproxy/tests/test_proxy.py | 9 --------- 3 files changed, 32 deletions(-) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index 2d44c0b14e1..58802fb07ec 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -4,7 +4,6 @@ import httpretty import httpretty.core import nose -import sys import datetime import pylons @@ -56,19 +55,12 @@ def __getattr__(self, attr): httpretty.core.fakesock.socket = HTTPrettyFix -# avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 -if sys.version_info < (2, 7, 0): - import socket - socket.setdefaulttimeout(1) - - class TestDatastoreCreate(tests.WsgiAppCase): sysadmin_user = None normal_user = None @classmethod def setup_class(cls): - wsgiapp = middleware.make_app(config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) if not tests.is_datastore_supported(): @@ -84,20 +76,11 @@ def setup_class(cls): set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) - # Httpretty crashes with Solr on Python 2.6, - # skip the tests - if (sys.version_info[0] == 2 and sys.version_info[1] == 6): - raise nose.SkipTest() - @classmethod def teardown_class(cls): rebuild_all_dbs(cls.Session) p.unload('datastore') p.unload('datapusher') - # Reenable Solr indexing - if (sys.version_info[0] == 2 and sys.version_info[1] == 6 - and not p.plugin_loaded('synchronous_search')): - p.load('synchronous_search') def test_create_ckan_resource_in_package(self): package = model.Package.get('annakarenina') diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index a36340ec82d..8e97355b525 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -22,12 +22,6 @@ from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type -# avoid hanging tests https://github.com/gabrielfalcao/HTTPretty/issues/34 -if sys.version_info < (2, 7, 0): - import socket - socket.setdefaulttimeout(1) - - class TestDatastoreCreateNewTests(object): @classmethod def setup_class(cls): diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py index 28a382ef713..78413da7685 100644 --- a/ckanext/resourceproxy/tests/test_proxy.py +++ b/ckanext/resourceproxy/tests/test_proxy.py @@ -1,6 +1,5 @@ # encoding: utf-8 -import sys import requests import unittest import json @@ -60,20 +59,12 @@ def setup_class(cls): wsgiapp = middleware.make_app(config['global_conf'], **config) cls.app = paste.fixture.TestApp(wsgiapp) create_test_data.CreateTestData.create() - # Httpretty crashes with Solr on Python 2.6, - # skip the tests - if (sys.version_info[0] == 2 and sys.version_info[1] == 6): - raise nose.SkipTest() @classmethod def teardown_class(cls): config.clear() config.update(cls._original_config) model.repo.rebuild_db() - # Reenable Solr indexing - if (sys.version_info[0] == 2 and sys.version_info[1] == 6 - and not p.plugin_loaded('synchronous_search')): - p.load('synchronous_search') def setUp(self): self.url = 'http://www.ckan.org/static/example.json' From ff00d5c6fd79c80b7d6772b0cb578a8c584b2095 Mon Sep 17 00:00:00 2001 From: Deinok Date: Fri, 17 Jun 2016 20:50:31 +0200 Subject: [PATCH 071/210] PairTree and Readme fix --- README.rst | 4 ---- requirements.in | 2 +- requirements.txt | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index 2fadc2dd390..6f45f30f68a 100644 --- a/README.rst +++ b/README.rst @@ -20,10 +20,6 @@ CKAN: The Open Source Data Portal Software :target: https://coveralls.io/github/ckan/ckan?branch=master :alt: Coverage Status -.. image:: https://requires.io/github/ckan/ckan/requirements.svg?branch=master - :target: https://requires.io/github/ckan/ckan/requirements/?branch=master - :alt: Requirements Status - **CKAN is the world’s leading open-source data portal platform**. CKAN makes it easy to publish, share and work with data. It's a data management system that provides a powerful platform for cataloging, storing and accessing diff --git a/requirements.in b/requirements.in index 257251e5f08..c37be5373e8 100644 --- a/requirements.in +++ b/requirements.in @@ -9,7 +9,7 @@ Jinja2==2.8 Markdown==2.6.6 ofs==0.4.2 ordereddict==1.1 -Pairtree==0.5.8 +Pairtree==0.7.1-T passlib==1.6.5 paste==1.7.5.1 psycopg2==2.4.5 diff --git a/requirements.txt b/requirements.txt index 9e394c995f7..33b3ddf2a65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,7 @@ MarkupSafe==0.23 nose==1.3.7 # via pylons ofs==0.4.2 ordereddict==1.1 -Pairtree==0.5.8 +Pairtree==0.7.1-T passlib==1.6.5 paste==1.7.5.1 PasteDeploy==1.5.2 From ac69f064ce3a17aee89a1d4c8977fff4c718c700 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Mon, 20 Jun 2016 09:55:49 +0200 Subject: [PATCH 072/210] [#3126] only reference py2.7 in documentation --- doc/contributing/python.rst | 2 +- doc/maintaining/installing/install-from-source.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/contributing/python.rst b/doc/contributing/python.rst index b557dae81ff..58509d4f916 100644 --- a/doc/contributing/python.rst +++ b/doc/contributing/python.rst @@ -74,7 +74,7 @@ Imports Logging ------- -We use `the Python standard library's logging module `_ +We use `the Python standard library's logging module `_ to log messages in CKAN, e.g.:: import logging diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index 5dc82468219..f5f895333cf 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -33,7 +33,7 @@ wiki page for help): ===================== =============================================== Package Description ===================== =============================================== -Python `The Python programming language, v2.6 or 2.7 `_ +Python `The Python programming language, v2.7 `_ |postgres| `The PostgreSQL database system, v8.4 or newer `_ libpq `The C programmer's interface to PostgreSQL `_ pip `A tool for installing and managing Python packages `_ From 9ee75b530a687e43b9eee4fc049ef412b5ad432a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Mon, 20 Jun 2016 09:58:04 +0200 Subject: [PATCH 073/210] [#3126] remove py2.6 from paster templates --- ckan/pastertemplates/template/+dot+travis.yml_tmpl | 1 - ckan/pastertemplates/template/setup.py_tmpl | 1 - 2 files changed, 2 deletions(-) diff --git a/ckan/pastertemplates/template/+dot+travis.yml_tmpl b/ckan/pastertemplates/template/+dot+travis.yml_tmpl index 536ae829969..cc4b3d7cd1f 100644 --- a/ckan/pastertemplates/template/+dot+travis.yml_tmpl +++ b/ckan/pastertemplates/template/+dot+travis.yml_tmpl @@ -1,7 +1,6 @@ language: python sudo: required python: - - "2.6" - "2.7" env: PGVERSION=9.1 install: diff --git a/ckan/pastertemplates/template/setup.py_tmpl b/ckan/pastertemplates/template/setup.py_tmpl index 844042b0035..d6dafcc4cd5 100644 --- a/ckan/pastertemplates/template/setup.py_tmpl +++ b/ckan/pastertemplates/template/setup.py_tmpl @@ -43,7 +43,6 @@ setup( # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. - 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', ], From cd340b172e012020f86fdb712ead54c7d0f35460 Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 20 Jun 2016 12:48:41 +0200 Subject: [PATCH 074/210] Beaker Upgrade - Changed Comment --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index c37be5373e8..6839d570513 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,7 @@ # The file contains the direct ckan requirements. # Use pip-compile to create a requirements.txt file from this Babel==2.3.4 -Beaker==1.7.0 # need to pin this because of https://github.com/bbangert/beaker/commit/fc511ceda4305fcf54919b3f081fed7790e2fef3 +Beaker==1.8.0 # Needs to be pinned to a more up to date version than the Pylons one bleach==1.4.3 fanstatic==0.12 Flask==0.10.1 diff --git a/requirements.txt b/requirements.txt index 33b3ddf2a65..9fa62feb746 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # argparse==1.4.0 # via ofs Babel==2.3.4 -Beaker==1.7.0 +Beaker==1.8.0 bleach==1.4.3 decorator==4.0.6 # via pylons, sqlalchemy-migrate fanstatic==0.12 From afba1b078f96662f1e101ead85b9e41a25c2cca6 Mon Sep 17 00:00:00 2001 From: Raul Hidalgo Caballero Date: Mon, 20 Jun 2016 13:01:08 +0200 Subject: [PATCH 075/210] SimpleJson - Hand-fix deleted --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9fa62feb746..db90589524c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,7 +39,7 @@ repoze.who-friendlyform==1.0.8 repoze.who==2.3 requests==2.10.0 Routes==1.13 -simplejson==3.8.2 # via pylons HAND-FIXED FOR NOW #2681 +simplejson==3.8.2 # via pylons six==1.10.0 # via bleach, html5lib, pastescript, sqlalchemy-migrate sqlalchemy-migrate==0.10.0 SQLAlchemy==0.9.6 From 1fe29c72b8770a3bc7db097fe2e1f44fd53da984 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Tue, 21 Jun 2016 09:14:38 +0200 Subject: [PATCH 076/210] [#3128] Render resource view descriptions as Markdown. Previously, CKAN rendered resource view descriptions as plain text, in contrast to the help text given in the resource view edit form which says that Markdown is supported. --- ckan/templates/package/snippets/resource_view.html | 2 +- ckan/tests/controllers/test_package.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ckan/templates/package/snippets/resource_view.html b/ckan/templates/package/snippets/resource_view.html index 77cf21a4e82..93f1373bc9d 100644 --- a/ckan/templates/package/snippets/resource_view.html +++ b/ckan/templates/package/snippets/resource_view.html @@ -10,7 +10,7 @@ {{ _("Embed") }} -

{{ resource_view['description'] }}

+

{{ h.render_markdown(resource_view['description']) }}

{% if not to_preview and h.resource_view_is_filterable(resource_view) %} {% snippet 'package/snippets/resource_view_filters.html', resource=resource %} diff --git a/ckan/tests/controllers/test_package.py b/ckan/tests/controllers/test_package.py index aac6ba8b864..9275be446bc 100644 --- a/ckan/tests/controllers/test_package.py +++ b/ckan/tests/controllers/test_package.py @@ -1050,6 +1050,17 @@ def test_inexistent_resource_view_page_returns_not_found_code(self): app = self._get_test_app() app.get(url, status=404) + def test_resource_view_description_is_rendered_as_markdown(self): + resource_view = factories.ResourceView(description="Some **Markdown**") + url = url_for(controller='package', + action='resource_read', + id=resource_view['package_id'], + resource_id=resource_view['resource_id'], + view_id=resource_view['id']) + app = self._get_test_app() + response = app.get(url) + response.mustcontain('Some Markdown') + class TestResourceRead(helpers.FunctionalTestBase): @classmethod From dfd2a9187cde6161fc85ba2aba5feefa2516e227 Mon Sep 17 00:00:00 2001 From: Carl Lange Date: Wed, 22 Jun 2016 00:01:59 +0100 Subject: [PATCH 077/210] Fix Pep8 error (long comment) --- ckanext/datapusher/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/datapusher/cli.py b/ckanext/datapusher/cli.py index 2328b051f14..c5d098c9671 100644 --- a/ckanext/datapusher/cli.py +++ b/ckanext/datapusher/cli.py @@ -63,7 +63,8 @@ def _resubmit_all(self): def _submit_all_packages(self): # submit every package - # for each package in the package list, submit each resource w/ _submit_package + # for each package in the package list, + # submit each resource w/ _submit_package import ckan.model as model package_list = p.toolkit.get_action('package_list') for p_id in package_list({'model': model, 'ignore_auth': True}, {}): From 8396d7c6090defefd5f515054afac9362c70c6ff Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Fri, 17 Jun 2016 16:15:33 +0200 Subject: [PATCH 078/210] Add encoding specification. --- bin/running_stats.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/running_stats.py b/bin/running_stats.py index 006ce6cc029..a2612f35d0f 100644 --- a/bin/running_stats.py +++ b/bin/running_stats.py @@ -1,3 +1,5 @@ +# encoding: utf-8 + '''Tool for a script to keep track changes performed on a large number of objects. From e7513d1df9bfa0b90347216e7199e3e58b23ed5b Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Fri, 17 Jun 2016 16:19:03 +0200 Subject: [PATCH 079/210] Add test to ensure string literals have a `u`, `b` or `ur` prefix. All existing Python source code files are explicitly white-listed. Over time, these should be fixed and delisted. New files should not be added to the list. --- ckan/tests/test_coding_standards.py | 689 +++++++++++++++++++++++++--- 1 file changed, 635 insertions(+), 54 deletions(-) diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py index 6010449676d..d5129476756 100644 --- a/ckan/tests/test_coding_standards.py +++ b/ckan/tests/test_coding_standards.py @@ -1,6 +1,6 @@ # encoding: utf-8 -'''A module for coding standards tests. +u'''A module for coding standards tests. These are tests that are not functional- or unit-testing any particular piece of CKAN code, but are checking coding standards. For example: checking that @@ -9,15 +9,55 @@ ''' +import ast import io import os import os.path import re import subprocess +import sys + + +FILESYSTEM_ENCODING = unicode(sys.getfilesystemencoding() + or sys.getdefaultencoding()) + +HERE = os.path.abspath(os.path.dirname(__file__.decode(FILESYSTEM_ENCODING))) + +PROJECT_ROOT = os.path.normpath(os.path.join(HERE, u'..', u'..')) + +# Directories which are ignored when checking Python source code files +IGNORED_DIRS = [ + u'ckan/include', +] + + +def walk_python_files(): + u''' + Generator that yields all CKAN Python source files. + + Yields 2-tuples containing the filename in absolute and relative (to + the project root) form. + ''' + def _is_dir_ignored(root, d): + if d.startswith(u'.'): + return True + return os.path.join(rel_root, d) in IGNORED_DIRS + + for abs_root, dirnames, filenames in os.walk(PROJECT_ROOT): + rel_root = os.path.relpath(abs_root, PROJECT_ROOT) + if rel_root == u'.': + rel_root = u'' + dirnames[:] = [d for d in dirnames if not _is_dir_ignored(rel_root, d)] + for filename in filenames: + if not filename.endswith(u'.py'): + continue + abs_name = os.path.join(abs_root, filename) + rel_name = os.path.join(rel_root, filename) + yield abs_name, rel_name def test_building_the_docs(): - '''There should be no warnings or errors when building the Sphinx docs. + u'''There should be no warnings or errors when building the Sphinx docs. This test unfortunately does take quite a long time to run - rebuilding the docs from scratch just takes a long time. @@ -27,38 +67,38 @@ def test_building_the_docs(): ''' try: output = subprocess.check_output( - ['python', - 'setup.py', - 'build_sphinx', - '--all-files', - '--fresh-env'], + [b'python', + b'setup.py', + b'build_sphinx', + b'--all-files', + b'--fresh-env'], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as err: assert False, ( - "Building the docs failed with return code: {code}".format( + u"Building the docs failed with return code: {code}".format( code=err.returncode)) - output_lines = output.split('\n') + output_lines = output.split(u'\n') - errors = [line for line in output_lines if 'ERROR' in line] + errors = [line for line in output_lines if u'ERROR' in line] if errors: - assert False, ("Don't add any errors to the Sphinx build: " - "{errors}".format(errors=errors)) + assert False, (u"Don't add any errors to the Sphinx build: " + u"{errors}".format(errors=errors)) - warnings = [line for line in output_lines if 'WARNING' in line] + warnings = [line for line in output_lines if u'WARNING' in line] # Some warnings have been around for a long time and aren't easy to fix. # These are allowed, but no more should be added. allowed_warnings = [ - 'WARNING: duplicate label ckan.auth.create_user_via_web', - 'WARNING: duplicate label ckan.auth.create_unowned_dataset', - 'WARNING: duplicate label ckan.auth.user_create_groups', - 'WARNING: duplicate label ckan.auth.anon_create_dataset', - 'WARNING: duplicate label ckan.auth.user_delete_organizations', - 'WARNING: duplicate label ckan.auth.create_user_via_api', - 'WARNING: duplicate label ckan.auth.create_dataset_if_not_in_organization', - 'WARNING: duplicate label ckan.auth.user_delete_groups', - 'WARNING: duplicate label ckan.auth.user_create_organizations', - 'WARNING: duplicate label ckan.auth.roles_that_cascade_to_sub_groups' + u'WARNING: duplicate label ckan.auth.create_user_via_web', + u'WARNING: duplicate label ckan.auth.create_unowned_dataset', + u'WARNING: duplicate label ckan.auth.user_create_groups', + u'WARNING: duplicate label ckan.auth.anon_create_dataset', + u'WARNING: duplicate label ckan.auth.user_delete_organizations', + u'WARNING: duplicate label ckan.auth.create_user_via_api', + u'WARNING: duplicate label ckan.auth.create_dataset_if_not_in_organization', + u'WARNING: duplicate label ckan.auth.user_delete_groups', + u'WARNING: duplicate label ckan.auth.user_create_organizations', + u'WARNING: duplicate label ckan.auth.roles_that_cascade_to_sub_groups' ] # Remove the allowed warnings from the list of collected warnings. @@ -73,56 +113,597 @@ def test_building_the_docs(): if warning not in warnings_to_remove] if new_warnings: - assert False, ("Don't add any new warnings to the Sphinx build: " - "{warnings}".format(warnings=new_warnings)) + assert False, (u"Don't add any new warnings to the Sphinx build: " + u"{warnings}".format(warnings=new_warnings)) def test_source_files_specify_encoding(): - ''' + u''' Test that *.py files have a PEP 263 UTF-8 encoding specification. Empty files and files that only contain comments are ignored. ''' - root_dir = os.path.join(os.path.dirname(__file__), '..', '..') - test_dirs = ['ckan', 'ckanext'] - ignored_dirs = ['ckan/include'] - pattern = re.compile(r'#.*?coding[:=][ \t]*utf-?8') + pattern = re.compile(ur'#.*?coding[:=][ \t]*utf-?8') decode_errors = [] no_specification = [] - - def check_file(filename): + for abs_path, rel_path in walk_python_files(): try: - with io.open(filename, encoding='utf-8') as f: + with io.open(abs_path, encoding=u'utf-8') as f: for line in f: line = line.strip() if pattern.match(line): # Pattern found - return - elif line and not line.startswith('#'): + break + elif line and not line.startswith(u'#'): # File contains non-empty non-comment line - no_specification.append(os.path.relpath(filename, - root_dir)) - return + no_specification.append(rel_path) + break except UnicodeDecodeError: - decode_errors.append(filename) - - for test_dir in test_dirs: - base_dir = os.path.join(root_dir, test_dir) - for root, dirnames, filenames in os.walk(base_dir): - dirnames[:] = [d for d in dirnames if not - os.path.relpath(os.path.join(root, d), root_dir) - in ignored_dirs] - for filename in filenames: - if not filename.endswith('.py'): - continue - check_file(os.path.join(root, filename)) + decode_errors.append(rel_path) msgs = [] if no_specification: - msgs.append('The following files are missing an encoding ' - + 'specification: {}'.format(no_specification)) + msgs.append(u'The following files are missing an encoding ' + + u'specification: {}'.format(no_specification)) if decode_errors: - msgs.append('The following files are not valid UTF-8: {}'.format( + msgs.append(u'The following files are not valid UTF-8: {}'.format( decode_errors)) if msgs: - assert False, '\n\n'.join(msgs) + assert False, u'\n\n'.join(msgs) + + +def renumerate(it): + u''' + Reverse enumerate. + + Yields tuples ``(i, x)`` where ``x`` are the items of ``it`` in + reverse order and ``i`` is the corresponding (decreasing) index. + ``it`` must support ``len``. + ''' + return zip(xrange(len(it) - 1, -1, -1), reversed(it)) + + +def find_unprefixed_string_literals(filename): + u''' + Find unprefixed string literals in a Python source file. + + Returns a list of ``(line_number, column)`` tuples (both 1-based) of + positions where string literals without a ``u`` or ``b`` prefix + start. + + Note: Due to limitations in Python's ``ast`` module this does not + check the rear parts of auto-concatenated string literals + (``'foo' 'bar'``). + ''' + with io.open(filename, encoding=u'utf-8') as f: + lines = f.readlines() + # In some versions of Python, the ast module cannot deal with + # encoding declarations (http://bugs.python.org/issue22221). We + # therefore replace all comment lines at the beginning of the file + # with empty lines (to keep the line numbers correct). + for i, line in enumerate(lines): + line = line.strip() + if line.startswith(u'#'): + lines[i] = u'\n' + elif line: + break + root = ast.parse(u''.join(lines), filename.encode(FILESYSTEM_ENCODING)) + problems = [] + for node in ast.walk(root): + if isinstance(node, ast.Str): + lineno = node.lineno - 1 + col_offset = node.col_offset + if col_offset == -1: + # `lineno` and `col_offset` are broken for literals that span + # multiple lines: For these, `lineno` contains the line of the + # *closing* quotes, and `col_offset` is always -1, see + # https://bugs.python.org/issue16806. We therefore have to + # find the start of the literal manually, which is difficult + # since '''-literals can contain """ and vice versa. The + # following code assumes that no ''' or """ literal begins on + # the same line where a multi-line literal ends. + last_line = lines[lineno] + if last_line.rfind(u'"""') > last_line.rfind(u"'''"): + quotes = u'"""' + else: + quotes = u"'''" + for lineno, line in renumerate(lines[:lineno]): + try: + i = line.rindex(quotes) + if (i > 1) and (line[i - 2:i].lower() == u'ur'): + col_offset = i - 2 + elif (i > 0) and (line[i - 1].lower() in u'rbu'): + col_offset = i - 1 + else: + col_offset = 0 + break + except ValueError: + continue + first_char = lines[lineno][col_offset] + if first_char not in u'ub': # Don't allow capital U and B either + problems.append((lineno + 1, col_offset + 1)) + return sorted(problems) + + +# List of files white-listed for the string literal prefix test. Files on the +# list are expected to be fixed over time and removed from the list. DO NOT ADD +# NEW FILES TO THE LIST. +_STRING_LITERALS_WHITELIST = [ + u'bin/running_stats.py', + u'ckan/__init__.py', + u'ckan/authz.py', + u'ckan/ckan_nose_plugin.py', + u'ckan/config/environment.py', + u'ckan/config/install.py', + u'ckan/config/middleware/__init__.py', + u'ckan/config/middleware/common_middleware.py', + u'ckan/config/middleware/flask_app.py', + u'ckan/config/middleware/pylons_app.py', + u'ckan/config/routing.py', + u'ckan/controllers/admin.py', + u'ckan/controllers/api.py', + u'ckan/controllers/error.py', + u'ckan/controllers/feed.py', + u'ckan/controllers/group.py', + u'ckan/controllers/home.py', + u'ckan/controllers/organization.py', + u'ckan/controllers/package.py', + u'ckan/controllers/partyline.py', + u'ckan/controllers/revision.py', + u'ckan/controllers/storage.py', + u'ckan/controllers/tag.py', + u'ckan/controllers/template.py', + u'ckan/controllers/user.py', + u'ckan/controllers/util.py', + u'ckan/exceptions.py', + u'ckan/i18n/check_po_files.py', + u'ckan/lib/activity_streams.py', + u'ckan/lib/activity_streams_session_extension.py', + u'ckan/lib/alphabet_paginate.py', + u'ckan/lib/app_globals.py', + u'ckan/lib/auth_tkt.py', + u'ckan/lib/authenticator.py', + u'ckan/lib/base.py', + u'ckan/lib/captcha.py', + u'ckan/lib/celery_app.py', + u'ckan/lib/cli.py', + u'ckan/lib/config_tool.py', + u'ckan/lib/create_test_data.py', + u'ckan/lib/datapreview.py', + u'ckan/lib/dictization/__init__.py', + u'ckan/lib/dictization/model_dictize.py', + u'ckan/lib/dictization/model_save.py', + u'ckan/lib/email_notifications.py', + u'ckan/lib/extract.py', + u'ckan/lib/fanstatic_extensions.py', + u'ckan/lib/fanstatic_resources.py', + u'ckan/lib/formatters.py', + u'ckan/lib/hash.py', + u'ckan/lib/helpers.py', + u'ckan/lib/i18n.py', + u'ckan/lib/jinja_extensions.py', + u'ckan/lib/jsonp.py', + u'ckan/lib/mailer.py', + u'ckan/lib/maintain.py', + u'ckan/lib/munge.py', + u'ckan/lib/navl/__init__.py', + u'ckan/lib/navl/dictization_functions.py', + u'ckan/lib/navl/validators.py', + u'ckan/lib/plugins.py', + u'ckan/lib/render.py', + u'ckan/lib/search/__init__.py', + u'ckan/lib/search/common.py', + u'ckan/lib/search/index.py', + u'ckan/lib/search/query.py', + u'ckan/lib/search/sql.py', + u'ckan/lib/uploader.py', + u'ckan/logic/__init__.py', + u'ckan/logic/action/__init__.py', + u'ckan/logic/action/create.py', + u'ckan/logic/action/delete.py', + u'ckan/logic/action/get.py', + u'ckan/logic/action/patch.py', + u'ckan/logic/action/update.py', + u'ckan/logic/auth/__init__.py', + u'ckan/logic/auth/create.py', + u'ckan/logic/auth/delete.py', + u'ckan/logic/auth/get.py', + u'ckan/logic/auth/patch.py', + u'ckan/logic/auth/update.py', + u'ckan/logic/converters.py', + u'ckan/logic/schema.py', + u'ckan/logic/validators.py', + u'ckan/migration/manage.py', + u'ckan/migration/versions/001_add_existing_tables.py', + u'ckan/migration/versions/002_add_author_and_maintainer.py', + u'ckan/migration/versions/003_add_user_object.py', + u'ckan/migration/versions/004_add_group_object.py', + u'ckan/migration/versions/005_add_authorization_tables.py', + u'ckan/migration/versions/006_add_ratings.py', + u'ckan/migration/versions/007_add_system_roles.py', + u'ckan/migration/versions/008_update_vdm_ids.py', + u'ckan/migration/versions/009_add_creation_timestamps.py', + u'ckan/migration/versions/010_add_user_about.py', + u'ckan/migration/versions/011_add_package_search_vector.py', + u'ckan/migration/versions/012_add_resources.py', + u'ckan/migration/versions/013_add_hash.py', + u'ckan/migration/versions/014_hash_2.py', + u'ckan/migration/versions/015_remove_state_object.py', + u'ckan/migration/versions/016_uuids_everywhere.py', + u'ckan/migration/versions/017_add_pkg_relationships.py', + u'ckan/migration/versions/018_adjust_licenses.py', + u'ckan/migration/versions/019_pkg_relationships_state.py', + u'ckan/migration/versions/020_add_changeset.py', + u'ckan/migration/versions/022_add_group_extras.py', + u'ckan/migration/versions/023_add_harvesting.py', + u'ckan/migration/versions/024_add_harvested_document.py', + u'ckan/migration/versions/025_add_authorization_groups.py', + u'ckan/migration/versions/026_authorization_group_user_pk.py', + u'ckan/migration/versions/027_adjust_harvester.py', + u'ckan/migration/versions/028_drop_harvest_source_status.py', + u'ckan/migration/versions/029_version_groups.py', + u'ckan/migration/versions/030_additional_user_attributes.py', + u'ckan/migration/versions/031_move_openid_to_new_field.py', + u'ckan/migration/versions/032_add_extra_info_field_to_resources.py', + u'ckan/migration/versions/033_auth_group_user_id_add_conditional.py', + u'ckan/migration/versions/034_resource_group_table.py', + u'ckan/migration/versions/035_harvesting_doc_versioning.py', + u'ckan/migration/versions/036_lockdown_roles.py', + u'ckan/migration/versions/037_role_anon_editor.py', + u'ckan/migration/versions/038_delete_migration_tables.py', + u'ckan/migration/versions/039_add_expired_id_and_dates.py', + u'ckan/migration/versions/040_reset_key_on_user.py', + u'ckan/migration/versions/041_resource_new_fields.py', + u'ckan/migration/versions/042_user_revision_indexes.py', + u'ckan/migration/versions/043_drop_postgres_search.py', + u'ckan/migration/versions/044_add_task_status.py', + u'ckan/migration/versions/045_user_name_unique.py', + u'ckan/migration/versions/046_drop_changesets.py', + u'ckan/migration/versions/047_rename_package_group_member.py', + u'ckan/migration/versions/048_add_activity_streams_tables.py', + u'ckan/migration/versions/049_add_group_approval_status.py', + u'ckan/migration/versions/050_term_translation_table.py', + u'ckan/migration/versions/051_add_tag_vocabulary.py', + u'ckan/migration/versions/052_update_member_capacities.py', + u'ckan/migration/versions/053_add_group_logo.py', + u'ckan/migration/versions/054_add_resource_created_date.py', + u'ckan/migration/versions/055_update_user_and_activity_detail.py', + u'ckan/migration/versions/056_add_related_table.py', + u'ckan/migration/versions/057_tracking.py', + u'ckan/migration/versions/058_add_follower_tables.py', + u'ckan/migration/versions/059_add_related_count_and_flag.py', + u'ckan/migration/versions/060_add_system_info_table.py', + u'ckan/migration/versions/061_add_follower__group_table.py', + u'ckan/migration/versions/062_add_dashboard_table.py', + u'ckan/migration/versions/063_org_changes.py', + u'ckan/migration/versions/064_add_email_last_sent_column.py', + u'ckan/migration/versions/065_add_email_notifications_preference.py', + u'ckan/migration/versions/066_default_package_type.py', + u'ckan/migration/versions/067_turn_extras_to_strings.py', + u'ckan/migration/versions/068_add_package_extras_index.py', + u'ckan/migration/versions/069_resource_url_and_metadata_modified.py', + u'ckan/migration/versions/070_add_activity_and_resource_indexes.py', + u'ckan/migration/versions/071_add_state_column_to_user_table.py', + u'ckan/migration/versions/072_add_resource_view.py', + u'ckan/migration/versions/073_update_resource_view_resource_id_constraint.py', + u'ckan/migration/versions/074_remove_resource_groups.py', + u'ckan/migration/versions/075_rename_view_plugins.py', + u'ckan/migration/versions/076_rename_view_plugins_2.py', + u'ckan/migration/versions/077_add_revisions_to_system_info.py', + u'ckan/migration/versions/078_remove_old_authz_model.py', + u'ckan/migration/versions/079_resource_revision_index.py', + u'ckan/migration/versions/080_continuity_id_indexes.py', + u'ckan/migration/versions/081_set_datastore_active.py', + u'ckan/migration/versions/082_create_index_creator_user_id.py', + u'ckan/migration/versions/083_remove_related_items.py', + u'ckan/migration/versions/084_add_metadata_created.py', + u'ckan/model/__init__.py', + u'ckan/model/activity.py', + u'ckan/model/core.py', + u'ckan/model/dashboard.py', + u'ckan/model/domain_object.py', + u'ckan/model/extension.py', + u'ckan/model/follower.py', + u'ckan/model/group.py', + u'ckan/model/group_extra.py', + u'ckan/model/license.py', + u'ckan/model/meta.py', + u'ckan/model/misc.py', + u'ckan/model/modification.py', + u'ckan/model/package.py', + u'ckan/model/package_extra.py', + u'ckan/model/package_relationship.py', + u'ckan/model/rating.py', + u'ckan/model/resource.py', + u'ckan/model/resource_view.py', + u'ckan/model/system_info.py', + u'ckan/model/tag.py', + u'ckan/model/task_status.py', + u'ckan/model/term_translation.py', + u'ckan/model/tracking.py', + u'ckan/model/types.py', + u'ckan/model/user.py', + u'ckan/model/vocabulary.py', + u'ckan/pastertemplates/__init__.py', + u'ckan/plugins/core.py', + u'ckan/plugins/interfaces.py', + u'ckan/plugins/toolkit.py', + u'ckan/plugins/toolkit_sphinx_extension.py', + u'ckan/tests/config/test_environment.py', + u'ckan/tests/config/test_middleware.py', + u'ckan/tests/controllers/__init__.py', + u'ckan/tests/controllers/test_admin.py', + u'ckan/tests/controllers/test_api.py', + u'ckan/tests/controllers/test_feed.py', + u'ckan/tests/controllers/test_group.py', + u'ckan/tests/controllers/test_home.py', + u'ckan/tests/controllers/test_organization.py', + u'ckan/tests/controllers/test_package.py', + u'ckan/tests/controllers/test_tags.py', + u'ckan/tests/controllers/test_user.py', + u'ckan/tests/controllers/test_util.py', + u'ckan/tests/factories.py', + u'ckan/tests/helpers.py', + u'ckan/tests/i18n/test_check_po_files.py', + u'ckan/tests/legacy/__init__.py', + u'ckan/tests/legacy/ckantestplugins.py', + u'ckan/tests/legacy/functional/api/__init__.py', + u'ckan/tests/legacy/functional/api/base.py', + u'ckan/tests/legacy/functional/api/model/test_group.py', + u'ckan/tests/legacy/functional/api/model/test_licenses.py', + u'ckan/tests/legacy/functional/api/model/test_package.py', + u'ckan/tests/legacy/functional/api/model/test_ratings.py', + u'ckan/tests/legacy/functional/api/model/test_relationships.py', + u'ckan/tests/legacy/functional/api/model/test_revisions.py', + u'ckan/tests/legacy/functional/api/model/test_tag.py', + u'ckan/tests/legacy/functional/api/model/test_vocabulary.py', + u'ckan/tests/legacy/functional/api/test_activity.py', + u'ckan/tests/legacy/functional/api/test_api.py', + u'ckan/tests/legacy/functional/api/test_dashboard.py', + u'ckan/tests/legacy/functional/api/test_email_notifications.py', + u'ckan/tests/legacy/functional/api/test_follow.py', + u'ckan/tests/legacy/functional/api/test_misc.py', + u'ckan/tests/legacy/functional/api/test_package_search.py', + u'ckan/tests/legacy/functional/api/test_resource.py', + u'ckan/tests/legacy/functional/api/test_resource_search.py', + u'ckan/tests/legacy/functional/api/test_user.py', + u'ckan/tests/legacy/functional/api/test_util.py', + u'ckan/tests/legacy/functional/test_activity.py', + u'ckan/tests/legacy/functional/test_admin.py', + u'ckan/tests/legacy/functional/test_error.py', + u'ckan/tests/legacy/functional/test_group.py', + u'ckan/tests/legacy/functional/test_package.py', + u'ckan/tests/legacy/functional/test_pagination.py', + u'ckan/tests/legacy/functional/test_preview_interface.py', + u'ckan/tests/legacy/functional/test_revision.py', + u'ckan/tests/legacy/functional/test_tag.py', + u'ckan/tests/legacy/functional/test_tracking.py', + u'ckan/tests/legacy/functional/test_user.py', + u'ckan/tests/legacy/html_check.py', + u'ckan/tests/legacy/lib/__init__.py', + u'ckan/tests/legacy/lib/test_alphabet_pagination.py', + u'ckan/tests/legacy/lib/test_authenticator.py', + u'ckan/tests/legacy/lib/test_cli.py', + u'ckan/tests/legacy/lib/test_dictization.py', + u'ckan/tests/legacy/lib/test_dictization_schema.py', + u'ckan/tests/legacy/lib/test_email_notifications.py', + u'ckan/tests/legacy/lib/test_hash.py', + u'ckan/tests/legacy/lib/test_helpers.py', + u'ckan/tests/legacy/lib/test_i18n.py', + u'ckan/tests/legacy/lib/test_navl.py', + u'ckan/tests/legacy/lib/test_resource_search.py', + u'ckan/tests/legacy/lib/test_simple_search.py', + u'ckan/tests/legacy/lib/test_solr_package_search.py', + u'ckan/tests/legacy/lib/test_solr_package_search_synchronous_update.py', + u'ckan/tests/legacy/lib/test_solr_schema_version.py', + u'ckan/tests/legacy/lib/test_solr_search_index.py', + u'ckan/tests/legacy/lib/test_tag_search.py', + u'ckan/tests/legacy/logic/test_action.py', + u'ckan/tests/legacy/logic/test_auth.py', + u'ckan/tests/legacy/logic/test_init.py', + u'ckan/tests/legacy/logic/test_member.py', + u'ckan/tests/legacy/logic/test_tag.py', + u'ckan/tests/legacy/logic/test_tag_vocab.py', + u'ckan/tests/legacy/logic/test_validators.py', + u'ckan/tests/legacy/misc/test_format_text.py', + u'ckan/tests/legacy/misc/test_mock_mail_server.py', + u'ckan/tests/legacy/misc/test_sync.py', + u'ckan/tests/legacy/mock_mail_server.py', + u'ckan/tests/legacy/mock_plugin.py', + u'ckan/tests/legacy/models/test_activity.py', + u'ckan/tests/legacy/models/test_extras.py', + u'ckan/tests/legacy/models/test_follower.py', + u'ckan/tests/legacy/models/test_group.py', + u'ckan/tests/legacy/models/test_misc.py', + u'ckan/tests/legacy/models/test_package.py', + u'ckan/tests/legacy/models/test_package_relationships.py', + u'ckan/tests/legacy/models/test_purge_revision.py', + u'ckan/tests/legacy/models/test_resource.py', + u'ckan/tests/legacy/models/test_revision.py', + u'ckan/tests/legacy/models/test_user.py', + u'ckan/tests/legacy/pylons_controller.py', + u'ckan/tests/legacy/schema/test_schema.py', + u'ckan/tests/legacy/test_coding_standards.py', + u'ckan/tests/legacy/test_plugins.py', + u'ckan/tests/legacy/test_versions.py', + u'ckan/tests/lib/__init__.py', + u'ckan/tests/lib/dictization/test_model_dictize.py', + u'ckan/tests/lib/navl/test_dictization_functions.py', + u'ckan/tests/lib/navl/test_validators.py', + u'ckan/tests/lib/search/test_index.py', + u'ckan/tests/lib/test_app_globals.py', + u'ckan/tests/lib/test_auth_tkt.py', + u'ckan/tests/lib/test_base.py', + u'ckan/tests/lib/test_cli.py', + u'ckan/tests/lib/test_config_tool.py', + u'ckan/tests/lib/test_datapreview.py', + u'ckan/tests/lib/test_helpers.py', + u'ckan/tests/lib/test_mailer.py', + u'ckan/tests/lib/test_munge.py', + u'ckan/tests/lib/test_navl.py', + u'ckan/tests/logic/action/__init__.py', + u'ckan/tests/logic/action/test_create.py', + u'ckan/tests/logic/action/test_delete.py', + u'ckan/tests/logic/action/test_get.py', + u'ckan/tests/logic/action/test_patch.py', + u'ckan/tests/logic/action/test_update.py', + u'ckan/tests/logic/auth/__init__.py', + u'ckan/tests/logic/auth/test_create.py', + u'ckan/tests/logic/auth/test_delete.py', + u'ckan/tests/logic/auth/test_get.py', + u'ckan/tests/logic/auth/test_init.py', + u'ckan/tests/logic/auth/test_update.py', + u'ckan/tests/logic/test_conversion.py', + u'ckan/tests/logic/test_converters.py', + u'ckan/tests/logic/test_schema.py', + u'ckan/tests/logic/test_validators.py', + u'ckan/tests/migration/__init__.py', + u'ckan/tests/model/__init__.py', + u'ckan/tests/model/test_license.py', + u'ckan/tests/model/test_resource.py', + u'ckan/tests/model/test_resource_view.py', + u'ckan/tests/model/test_system_info.py', + u'ckan/tests/model/test_user.py', + u'ckan/tests/plugins/__init__.py', + u'ckan/tests/plugins/test_toolkit.py', + u'ckan/tests/test_authz.py', + u'ckan/tests/test_factories.py', + u'ckan/websetup.py', + u'ckanext/datapusher/cli.py', + u'ckanext/datapusher/helpers.py', + u'ckanext/datapusher/interfaces.py', + u'ckanext/datapusher/logic/action.py', + u'ckanext/datapusher/logic/schema.py', + u'ckanext/datapusher/plugin.py', + u'ckanext/datapusher/tests/test.py', + u'ckanext/datapusher/tests/test_action.py', + u'ckanext/datapusher/tests/test_default_views.py', + u'ckanext/datapusher/tests/test_interfaces.py', + u'ckanext/datastore/commands.py', + u'ckanext/datastore/controller.py', + u'ckanext/datastore/db.py', + u'ckanext/datastore/helpers.py', + u'ckanext/datastore/interfaces.py', + u'ckanext/datastore/logic/action.py', + u'ckanext/datastore/logic/auth.py', + u'ckanext/datastore/logic/schema.py', + u'ckanext/datastore/plugin.py', + u'ckanext/datastore/tests/helpers.py', + u'ckanext/datastore/tests/sample_datastore_plugin.py', + u'ckanext/datastore/tests/test_configure.py', + u'ckanext/datastore/tests/test_create.py', + u'ckanext/datastore/tests/test_db.py', + u'ckanext/datastore/tests/test_delete.py', + u'ckanext/datastore/tests/test_disable.py', + u'ckanext/datastore/tests/test_dump.py', + u'ckanext/datastore/tests/test_helpers.py', + u'ckanext/datastore/tests/test_info.py', + u'ckanext/datastore/tests/test_interface.py', + u'ckanext/datastore/tests/test_plugin.py', + u'ckanext/datastore/tests/test_search.py', + u'ckanext/datastore/tests/test_unit.py', + u'ckanext/datastore/tests/test_upsert.py', + u'ckanext/example_iauthfunctions/plugin_v2.py', + u'ckanext/example_iauthfunctions/plugin_v3.py', + u'ckanext/example_iauthfunctions/plugin_v4.py', + u'ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py', + u'ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py', + u'ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py', + u'ckanext/example_iconfigurer/controller.py', + u'ckanext/example_iconfigurer/plugin.py', + u'ckanext/example_iconfigurer/plugin_v1.py', + u'ckanext/example_iconfigurer/plugin_v2.py', + u'ckanext/example_iconfigurer/tests/test_example_iconfigurer.py', + u'ckanext/example_iconfigurer/tests/test_iconfigurer_toolkit.py', + u'ckanext/example_iconfigurer/tests/test_iconfigurer_update_config.py', + u'ckanext/example_idatasetform/plugin.py', + u'ckanext/example_idatasetform/plugin_v1.py', + u'ckanext/example_idatasetform/plugin_v2.py', + u'ckanext/example_idatasetform/plugin_v3.py', + u'ckanext/example_idatasetform/plugin_v4.py', + u'ckanext/example_idatasetform/tests/test_controllers.py', + u'ckanext/example_idatasetform/tests/test_example_idatasetform.py', + u'ckanext/example_igroupform/plugin.py', + u'ckanext/example_igroupform/tests/test_controllers.py', + u'ckanext/example_iresourcecontroller/plugin.py', + u'ckanext/example_iresourcecontroller/tests/test_example_iresourcecontroller.py', + u'ckanext/example_itemplatehelpers/plugin.py', + u'ckanext/example_itranslation/plugin.py', + u'ckanext/example_itranslation/plugin_v1.py', + u'ckanext/example_itranslation/tests/test_plugin.py', + u'ckanext/example_iuploader/plugin.py', + u'ckanext/example_iuploader/test/test_plugin.py', + u'ckanext/example_ivalidators/plugin.py', + u'ckanext/example_ivalidators/tests/test_ivalidators.py', + u'ckanext/example_theme/custom_config_setting/plugin.py', + u'ckanext/example_theme/custom_emails/plugin.py', + u'ckanext/example_theme/custom_emails/tests.py', + u'ckanext/example_theme/v01_empty_extension/plugin.py', + u'ckanext/example_theme/v02_empty_template/plugin.py', + u'ckanext/example_theme/v03_jinja/plugin.py', + u'ckanext/example_theme/v04_ckan_extends/plugin.py', + u'ckanext/example_theme/v05_block/plugin.py', + u'ckanext/example_theme/v06_super/plugin.py', + u'ckanext/example_theme/v07_helper_function/plugin.py', + u'ckanext/example_theme/v08_custom_helper_function/plugin.py', + u'ckanext/example_theme/v09_snippet/plugin.py', + u'ckanext/example_theme/v10_custom_snippet/plugin.py', + u'ckanext/example_theme/v11_HTML_and_CSS/plugin.py', + u'ckanext/example_theme/v12_extra_public_dir/plugin.py', + u'ckanext/example_theme/v13_custom_css/plugin.py', + u'ckanext/example_theme/v14_more_custom_css/plugin.py', + u'ckanext/example_theme/v15_fanstatic/plugin.py', + u'ckanext/example_theme/v16_initialize_a_javascript_module/plugin.py', + u'ckanext/example_theme/v17_popover/plugin.py', + u'ckanext/example_theme/v18_snippet_api/plugin.py', + u'ckanext/example_theme/v19_01_error/plugin.py', + u'ckanext/example_theme/v19_02_error_handling/plugin.py', + u'ckanext/example_theme/v20_pubsub/plugin.py', + u'ckanext/example_theme/v21_custom_jquery_plugin/plugin.py', + u'ckanext/imageview/plugin.py', + u'ckanext/imageview/tests/test_view.py', + u'ckanext/multilingual/plugin.py', + u'ckanext/multilingual/tests/test_multilingual_plugin.py', + u'ckanext/reclineview/plugin.py', + u'ckanext/reclineview/tests/test_view.py', + u'ckanext/resourceproxy/controller.py', + u'ckanext/resourceproxy/plugin.py', + u'ckanext/resourceproxy/tests/test_proxy.py', + u'ckanext/stats/__init__.py', + u'ckanext/stats/controller.py', + u'ckanext/stats/plugin.py', + u'ckanext/stats/stats.py', + u'ckanext/stats/tests/__init__.py', + u'ckanext/stats/tests/test_stats_lib.py', + u'ckanext/stats/tests/test_stats_plugin.py', + u'ckanext/test_tag_vocab_plugin.py', + u'ckanext/textview/plugin.py', + u'ckanext/textview/tests/test_view.py', + u'ckanext/webpageview/plugin.py', + u'ckanext/webpageview/tests/test_view.py', + u'doc/conf.py', + u'profile_tests.py', + u'setup.py', +] + + +def test_string_literals_are_prefixed(): + u''' + Test that string literals are prefixed by ``u``, ``b`` or ``ur``. + + See http://docs.ckan.org/en/latest/contributing/unicode.html. + ''' + errors = [] + for abs_path, rel_path in walk_python_files(): + if rel_path in _STRING_LITERALS_WHITELIST: + continue + problems = find_unprefixed_string_literals(abs_path) + if problems: + errors.append((rel_path, problems)) + if errors: + lines = [u'Unprefixed string literals:'] + for filename, problems in errors: + lines.append(u' ' + filename) + for line_no, col_no in problems: + lines.append(u' line {}, column {}'.format(line_no, col_no)) + raise AssertionError(u'\n'.join(lines)) From 2545f74e44aee3e80630f672ce76e74c46d1e101 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 23 Jun 2016 15:38:05 +0100 Subject: [PATCH 080/210] [#3132] Clarify package_relationship_update docs. This action is limited to only updating the comment property. subject, object and type are required to identify the relationship to update. --- ckan/logic/action/update.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index ae8bf610559..82e2e7f9124 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -385,22 +385,23 @@ def _update_package_relationship(relationship, comment, context): ref_package_by=ref_package_by) return rel_dict + def package_relationship_update(context, data_dict): '''Update a relationship between two datasets (packages). + The subject, object and type parameters are required to identify the + relationship. Only the comment can be updated. + You must be authorized to edit both the subject and the object datasets. - :param id: the id of the package relationship to update - :type id: string :param subject: the name or id of the dataset that is the subject of the - relationship (optional) + relationship :type subject: string :param object: the name or id of the dataset that is the object of the - relationship (optional) + relationship :param type: the type of the relationship, one of ``'depends_on'``, ``'dependency_of'``, ``'derives_from'``, ``'has_derivation'``, ``'links_to'``, ``'linked_from'``, ``'child_of'`` or ``'parent_of'`` - (optional) :type type: string :param comment: a comment about the relationship (optional) :type comment: string @@ -410,7 +411,8 @@ def package_relationship_update(context, data_dict): ''' model = context['model'] - schema = context.get('schema') or schema_.default_update_relationship_schema() + schema = context.get('schema') \ + or schema_.default_update_relationship_schema() id, id2, rel = _get_or_bust(data_dict, ['subject', 'object', 'type']) From bcc47010f0ba4d4a2718ffb3ad510701fc70a060 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 24 Jun 2016 14:07:50 +0100 Subject: [PATCH 081/210] [#2661] Add revision if owner-org updated. package_owner_org_update is called during a package_update to handle updating the owner organization. A package revision is created during this by package_update. If package_owner_org_update is called in isolation, no revision is created and an error occurs in vdm. This commit ensures a revision is created when package_owner_org_update is called outside of a package_update. --- ckan/logic/action/update.py | 13 +++++- ckan/tests/logic/action/test_update.py | 60 ++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index ae8bf610559..3111b397e12 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -292,6 +292,7 @@ def package_update(context, data_dict): context_org_update = context.copy() context_org_update['ignore_auth'] = True context_org_update['defer_commit'] = True + context_org_update['add_revision'] = False _get_action('package_owner_org_update')(context_org_update, {'id': pkg.id, 'organization_id': pkg.owner_org}) @@ -1020,6 +1021,7 @@ def package_owner_org_update(context, data_dict): :type id: string ''' model = context['model'] + user = context['user'] name_or_id = data_dict.get('id') organization_id = data_dict.get('organization_id') @@ -1039,10 +1041,17 @@ def package_owner_org_update(context, data_dict): org = None pkg.owner_org = None + if context.get('add_revision', True): + rev = model.repo.new_revision() + rev.author = user + if 'message' in context: + rev.message = context['message'] + else: + rev.message = _(u'REST API: Update object %s') % pkg.get("name") members = model.Session.query(model.Member) \ - .filter(model.Member.table_id == pkg.id) \ - .filter(model.Member.capacity == 'organization') + .filter(model.Member.table_id == pkg.id) \ + .filter(model.Member.capacity == 'organization') need_update = True for member_obj in members: diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 73e766368ef..00813019e47 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -858,3 +858,63 @@ def test_user_create_password_hash_not_for_normal_users(self): user_obj = model.User.get(user['id']) assert user_obj.password != 'pretend-this-is-a-valid-hash' + + +class TestPackageOwnerOrgUpdate(object): + + @classmethod + def teardown_class(cls): + helpers.reset_db() + + def setup(self): + helpers.reset_db() + + def test_package_owner_org_added(self): + '''A package without an owner_org can have one added.''' + sysadmin = factories.Sysadmin() + org = factories.Organization() + dataset = factories.Dataset() + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] is None + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=org['id']) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org == org['id'] + + def test_package_owner_org_changed(self): + '''A package with an owner_org can have it changed.''' + + sysadmin = factories.Sysadmin() + org_1 = factories.Organization() + org_2 = factories.Organization() + dataset = factories.Dataset(owner_org=org_1['id']) + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] == org_1['id'] + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=org_2['id']) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org == org_2['id'] + + def test_package_owner_org_removed(self): + '''A package with an owner_org can have it removed.''' + sysadmin = factories.Sysadmin() + org = factories.Organization() + dataset = factories.Dataset(owner_org=org['id']) + context = { + 'user': sysadmin['name'], + } + assert dataset['owner_org'] == org['id'] + helpers.call_action('package_owner_org_update', + context=context, + id=dataset['id'], + organization_id=None) + dataset_obj = model.Package.get(dataset['id']) + assert dataset_obj.owner_org is None From f240a8cf79e4a8d3a2a592f4cd3d6fa934a23ff7 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 24 Jun 2016 15:34:16 +0100 Subject: [PATCH 082/210] [#2661] Add 'add_revision' to ctx in pkg create. package_create also calls package_owner_org_update, and requires the 'add_revision' property adding to the context to prevent package_owner_org_update from creating an unecessary revision. --- ckan/logic/action/create.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 0a2a175309d..b85ef81f27b 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -197,6 +197,7 @@ def package_create(context, data_dict): context_org_update = context.copy() context_org_update['ignore_auth'] = True context_org_update['defer_commit'] = True + context_org_update['add_revision'] = False _get_action('package_owner_org_update')(context_org_update, {'id': pkg.id, 'organization_id': pkg.owner_org}) From 0b8f4d4e480f6d002369876e4e0bcc12557ddb92 Mon Sep 17 00:00:00 2001 From: Nick Such Date: Mon, 27 Jun 2016 13:10:32 -0400 Subject: [PATCH 083/210] Fix broken link to Google's Python style guide --- doc/contributing/python.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing/python.rst b/doc/contributing/python.rst index 58509d4f916..3e178a6aa2f 100644 --- a/doc/contributing/python.rst +++ b/doc/contributing/python.rst @@ -9,7 +9,7 @@ For Python code style follow `PEP 8`_ plus the guidelines below. Some good links about Python code style: - `Guide to Python `_ from Hitchhiker's -- `Google Python Style Guide `_ +- `Google Python Style Guide `_ .. seealso:: From 9548412999ee3710b3f8d1887625cb99848f9247 Mon Sep 17 00:00:00 2001 From: Michael Fincham Date: Tue, 28 Jun 2016 13:12:59 +1200 Subject: [PATCH 084/210] Change all example uses of domain names to be RFC compliant special-use names --- ckan/config/deployment.ini_tmpl | 6 +++--- doc/maintaining/configuration.rst | 10 +++++----- test.ini | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d25c9773804..50c58ecedbf 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -166,11 +166,11 @@ ckan.hide_activity_from_users = %(ckan.site_id)s ## Email settings -#email_to = you@yourdomain.com -#error_email_from = paste@localhost +#email_to = errors@example.com +#error_email_from = ckan-errors@example.com #smtp.server = localhost #smtp.starttls = False -#smtp.user = your_username@gmail.com +#smtp.user = username@example.com #smtp.password = your_password #smtp.mail_from = diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index 57b8503af2a..67e1521f56b 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -1775,7 +1775,7 @@ smtp.server Example:: - smtp.server = smtp.gmail.com:587 + smtp.server = smtp.example.com:587 Default value: ``None`` @@ -1801,7 +1801,7 @@ smtp.user Example:: - smtp.user = your_username@gmail.com + smtp.user = username@example.com Default value: ``None`` @@ -1827,7 +1827,7 @@ smtp.mail_from Example:: - smtp.mail_from = you@yourdomain.com + smtp.mail_from = ckan@example.com Default value: ``None`` @@ -1841,7 +1841,7 @@ email_to Example:: - email_to = you@yourdomain.com + email_to = errors@example.com Default value: ``None`` @@ -1854,7 +1854,7 @@ error_email_from Example:: - error_email_from = paste@localhost + error_email_from = ckan-errors@example.com Default value: ``None`` diff --git a/test.ini b/test.ini index 4578a15b255..2cf9d6722c7 100644 --- a/test.ini +++ b/test.ini @@ -6,9 +6,9 @@ [DEFAULT] debug = true # Uncomment and replace with the address which should receive any error reports -#email_to = you@yourdomain.com +#email_to = errors@example.com smtp_server = localhost -error_email_from = paste@localhost +error_email_from = ckan-errors@example.com [server:main] use = egg:Paste#http From f58e6841d6d6661417a6602b05a6fdc6441908d3 Mon Sep 17 00:00:00 2001 From: Sergey Motornyuk Date: Wed, 29 Jun 2016 12:00:06 +0300 Subject: [PATCH 085/210] Fix autocomplete request url. Wrapped previously hardcoded url into ckan.url function --- .../base/javascript/modules/resource-view-filters-form.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/public/base/javascript/modules/resource-view-filters-form.js b/ckan/public/base/javascript/modules/resource-view-filters-form.js index aa546285b00..2a7cb3f751c 100644 --- a/ckan/public/base/javascript/modules/resource-view-filters-form.js +++ b/ckan/public/base/javascript/modules/resource-view-filters-form.js @@ -10,7 +10,7 @@ ckan.module('resource-view-filters-form', function (jQuery) { width: 'resolve', minimumInputLength: 0, ajax: { - url: '/api/3/action/datastore_search', + url: ckan.url('/api/3/action/datastore_search'), datatype: 'json', quietMillis: 200, cache: true, From 644144a15a40962d783fc10f77c041055a4d9d0b Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 1 Jul 2016 15:19:27 +0100 Subject: [PATCH 086/210] [#3148] Remove WSGI Party When the App Dispatcher middleware was implemented on #2905 we used WSGI Party to allow communication between the Pylons and Flask applications. After having worked extensively on the Flask POC branches this has turned out to be unnecessary and to add quite a lot of complexity, specially with the language used around parties, invites, etc. To set it up there were two internal requests that were done while the app was being initialized, which were really confusing when trying to debug things. We didn't find any use case for having the two applications actually talk to each other, and the asking each app if they can handle the incoming request can be done just by calling a method directly. This commit removes all WSGI Party logic and its requirement, and extends the Flask and PylonsApp app objects with our own can_handle_request(environ) method, which is directly called by AskAppDispatcherMiddleware on each request. --- ckan/config/middleware/__init__.py | 46 ++++++-------- ckan/config/middleware/flask_app.py | 40 +++--------- ckan/config/middleware/pylons_app.py | 45 ++++++++++++- ckan/controllers/partyline.py | 66 ------------------- ckan/tests/config/test_middleware.py | 94 ++++++++++++++-------------- requirements.in | 1 - requirements.txt | 1 - 7 files changed, 116 insertions(+), 177 deletions(-) delete mode 100644 ckan/controllers/partyline.py diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py index 33adb1be313..25502f39d5c 100644 --- a/ckan/config/middleware/__init__.py +++ b/ckan/config/middleware/__init__.py @@ -5,7 +5,6 @@ import webob from werkzeug.test import create_environ, run_wsgi_app -from wsgi_party import WSGIParty from ckan.config.middleware.flask_app import make_flask_stack from ckan.config.middleware.pylons_app import make_pylons_stack @@ -51,11 +50,11 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): return app -class AskAppDispatcherMiddleware(WSGIParty): +class AskAppDispatcherMiddleware(object): ''' - Establish a 'partyline' to each provided app. Select which app to call - by asking each if they can handle the requested path at PATH_INFO. + Dispatches incoming requests to either the Flask or Pylons apps depending + on the WSGI environ. Used to help transition from Pylons to Flask, and should be removed once Pylons has been deprecated and all app requests are handled by Flask. @@ -73,40 +72,31 @@ class AskAppDispatcherMiddleware(WSGIParty): Flask Extension > Pylons Extension > Flask Core > Pylons Core ''' - def __init__(self, apps=None, invites=(), ignore_missing_services=False): + def __init__(self, apps=None): # Dict of apps managed by this middleware {: , ...} self.apps = apps or {} - # A dict of service name => handler mappings. - self.handlers = {} - - # If True, suppress :class:`NoSuchServiceName` errors. Default: False. - self.ignore_missing_services = ignore_missing_services - - self.send_invitations(apps) + def ask_around(self, environ): + '''Checks with all apps whether they can handle the incoming request + ''' + answers = [ + app._wsgi_app.can_handle_request(environ) + for name, app in self.apps.iteritems() + ] + # Sort answers by app name + answers = sorted(answers, key=lambda x: x[1]) + log.debug('Route support answers for {0} {1}: {2}'.format( + environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), + answers)) - def send_invitations(self, apps): - '''Call each app at the invite route to establish a partyline. Called - on init.''' - PATH = '/__invite__/' - for app_name, app in apps.items(): - environ = create_environ(path=PATH) - environ[self.partyline_key] = self.operator_class(self) - # A reference to the handling app. Used to id the app when - # responding to a handling request. - environ['partyline_handling_app'] = app_name - run_wsgi_app(app, environ) + return answers def __call__(self, environ, start_response): '''Determine which app to call by asking each app if it can handle the url and method defined on the eviron''' - # :::TODO::: Enforce order of precedence for dispatching to apps here. app_name = 'pylons_app' # currently defaulting to pylons app - answers = self.ask_around('can_handle_request', environ) - log.debug('Route support answers for {0} {1}: {2}'.format( - environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), - answers)) + answers = self.ask_around(environ) available_handlers = [] for answer in answers: if len(answer) == 2: diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index 8d37e26d1b8..f88d576268a 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -1,13 +1,8 @@ # encoding: utf-8 from flask import Flask -from flask import abort as flask_abort -from flask import request as flask_request -from flask import _request_ctx_stack from werkzeug.exceptions import HTTPException -from wsgi_party import WSGIParty, HighAndDry - import logging log = logging.getLogger(__name__) @@ -17,7 +12,7 @@ def make_flask_stack(conf): """ This has to pass the flask app through all the same middleware that Pylons used """ - app = CKANFlask(__name__) + app = flask_app = CKANFlask(__name__) @app.route('/hello', methods=['GET']) def hello_world(): @@ -27,38 +22,19 @@ def hello_world(): def hello_world_post(): return 'Hello World, this was posted to Flask' + # Add a reference to the actual Flask app so it's easier to access + setattr(app, '_wsgi_app', flask_app) + return app class CKANFlask(Flask): - '''Extend the Flask class with a special view to join the 'partyline' - established by AskAppDispatcherMiddleware. - - Also provide a 'can_handle_request' method. + '''Extend the Flask class with a special method called on incoming + requests by AskAppDispatcherMiddleware. ''' - def __init__(self, import_name, *args, **kwargs): - super(CKANFlask, self).__init__(import_name, *args, **kwargs) - self.add_url_rule('/__invite__/', endpoint='partyline', - view_func=self.join_party) - self.partyline = None - self.partyline_connected = False - self.invitation_context = None - self.app_name = None # A label for the app handling this request - # (this app). - - def join_party(self, request=flask_request): - # Bootstrap, turn the view function into a 404 after registering. - if self.partyline_connected: - # This route does not exist at the HTTP level. - flask_abort(404) - self.invitation_context = _request_ctx_stack.top - self.partyline = request.environ.get(WSGIParty.partyline_key) - self.app_name = request.environ.get('partyline_handling_app') - self.partyline.connect('can_handle_request', self.can_handle_request) - self.partyline_connected = True - return 'ok' + app_name = 'flask_app' def can_handle_request(self, environ): ''' @@ -78,4 +54,4 @@ def can_handle_request(self, environ): endpoint, args)) return (True, self.app_name) except HTTPException: - raise HighAndDry() + return (False, self.app_name) diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py index 4d2af590c0c..7af95aec9c2 100644 --- a/ckan/config/middleware/pylons_app.py +++ b/ckan/config/middleware/pylons_app.py @@ -54,7 +54,8 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): load_environment(conf, app_conf) # The Pylons WSGI app - app = PylonsApp() + app = pylons_app = CKANPylonsApp() + # set pylons globals app_globals.reset() @@ -177,9 +178,51 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): app = common_middleware.RootPathMiddleware(app, config) + # Add a reference to the actual Pylons app so it's easier to access + setattr(app, '_wsgi_app', pylons_app) + return app +class CKANPylonsApp(PylonsApp): + + app_name = 'pylons_app' + + def can_handle_request(self, environ): + ''' + Decides whether it can handle a request with the Pylons app by + matching the request environ against the route mapper + + Returns (True, 'pylons_app', origin) if this is the case. + + origin can be either 'core' or 'extension' depending on where + the route was defined. + + NOTE: There is currently a catch all route for GET requests to + point arbitrary urls to templates with the same name: + + map.connect('/*url', controller='template', action='view') + + This means that this function will match all GET requests. This + does not cause issues as the Pylons core routes are the last to + take precedence so the current behaviour is kept, but it's worth + keeping in mind. + ''' + + pylons_mapper = config['routes.map'] + match_route = pylons_mapper.routematch(environ=environ) + if match_route: + match, route = match_route + origin = 'core' + if hasattr(route, '_ckan_core') and not route._ckan_core: + origin = 'extension' + log.debug('Pylons route match: {0} Origin: {1}'.format( + match, origin)) + return (True, self.app_name, origin) + else: + return (False, self.app_name) + + def generate_close_and_callback(iterable, callback, environ): """ return a generator that passes through items from iterable diff --git a/ckan/controllers/partyline.py b/ckan/controllers/partyline.py deleted file mode 100644 index 30dbf6edfba..00000000000 --- a/ckan/controllers/partyline.py +++ /dev/null @@ -1,66 +0,0 @@ -# encoding: utf-8 - -from pylons.controllers import WSGIController -from pylons import config - -import ckan.lib.base as base -from ckan.common import request - -from wsgi_party import WSGIParty, HighAndDry - -import logging -log = logging.getLogger(__name__) - - -class PartylineController(WSGIController): - - '''Handle requests from the WSGI stack 'partyline'. Most importantly, - answers the question, 'can you handle this url?'. ''' - - def __init__(self, *args, **kwargs): - super(PartylineController, self).__init__(*args, **kwargs) - self.app_name = None # A reference to the main pylons app. - self.partyline_connected = False - - def join_party(self): - if self.partyline_connected: - base.abort(404) - self.partyline = request.environ.get(WSGIParty.partyline_key) - self.app_name = request.environ.get('partyline_handling_app') - self.partyline.connect('can_handle_request', self.can_handle_request) - self.partyline_connected = True - return 'ok' - - def can_handle_request(self, environ): - ''' - Decides whether it can handle a request with the Pylons app by - matching the request environ against the route mapper - - Returns (True, 'pylons_app', origin) if this is the case. - - origin can be either 'core' or 'extension' depending on where - the route was defined. - - NOTE: There is currently a catch all route for GET requests to - point arbitrary urls to templates with the same name: - - map.connect('/*url', controller='template', action='view') - - This means that this function will match all GET requests. This - does not cause issues as the Pylons core routes are the last to - take precedence so the current behaviour is kept, but it's worth - keeping in mind. - ''' - - pylons_mapper = config['routes.map'] - match_route = pylons_mapper.routematch(environ=environ) - if match_route: - match, route = match_route - origin = 'core' - if hasattr(route, '_ckan_core') and not route._ckan_core: - origin = 'extension' - log.debug('Pylons route match: {0} Origin: {1}'.format( - match, origin)) - return (True, self.app_name, origin) - else: - raise HighAndDry() diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 45478e322b9..633f92d14f3 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -40,20 +40,6 @@ class TestAppDispatcherPlain(object): the mocks, so they don't extend FunctionalTestBase ''' - def test_invitations_are_sent(self): - - with mock.patch.object(AskAppDispatcherMiddleware, 'send_invitations') as \ - mock_send_invitations: - - # This will create the whole WSGI stack - helpers._get_test_app() - - assert mock_send_invitations.called - eq_(len(mock_send_invitations.call_args[0]), 1) - - eq_(sorted(mock_send_invitations.call_args[0][0].keys()), - ['flask_app', 'pylons_app']) - def test_flask_can_handle_request_is_called_with_environ(self): with mock.patch.object(CKANFlask, 'can_handle_request') as \ @@ -142,7 +128,7 @@ def test_ask_around_is_called_with_args(self): ckan_app(environ, start_response) assert mock_ask_around.called - mock_ask_around.assert_called_with('can_handle_request', environ) + mock_ask_around.assert_called_with(environ) def test_ask_around_flask_core_route_get(self): @@ -157,7 +143,7 @@ def test_ask_around_flask_core_route_get(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) # Even though this route is defined in Flask, there is catch all route # in Pylons for all requests to point arbitrary urls to templates with @@ -180,7 +166,7 @@ def test_ask_around_flask_core_route_post(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) # Even though this route is defined in Flask, there is catch all route # in Pylons for all requests to point arbitrary urls to templates with @@ -203,12 +189,14 @@ def test_ask_around_pylons_core_route_get(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) - eq_(len(answers), 1) - eq_(answers[0][0], True) - eq_(answers[0][1], 'pylons_app') - eq_(answers[0][2], 'core') + eq_(len(answers), 2) + eq_(answers[0][0], False) + eq_(answers[0][1], 'flask_app') + eq_(answers[1][0], True) + eq_(answers[1][1], 'pylons_app') + eq_(answers[1][2], 'core') def test_ask_around_pylons_core_route_post(self): @@ -223,12 +211,14 @@ def test_ask_around_pylons_core_route_post(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) - eq_(len(answers), 1) - eq_(answers[0][0], True) - eq_(answers[0][1], 'pylons_app') - eq_(answers[0][2], 'core') + eq_(len(answers), 2) + eq_(answers[0][0], False) + eq_(answers[0][1], 'flask_app') + eq_(answers[1][0], True) + eq_(answers[1][1], 'pylons_app') + eq_(answers[1][2], 'core') def test_ask_around_pylons_extension_route_get_before_map(self): @@ -246,12 +236,14 @@ def test_ask_around_pylons_extension_route_get_before_map(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) - eq_(len(answers), 1) - eq_(answers[0][0], True) - eq_(answers[0][1], 'pylons_app') - eq_(answers[0][2], 'extension') + eq_(len(answers), 2) + eq_(answers[0][0], False) + eq_(answers[0][1], 'flask_app') + eq_(answers[1][0], True) + eq_(answers[1][1], 'pylons_app') + eq_(answers[1][2], 'extension') p.unload('test_routing_plugin') @@ -271,12 +263,14 @@ def test_ask_around_pylons_extension_route_post(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) - eq_(len(answers), 1) - eq_(answers[0][0], True) - eq_(answers[0][1], 'pylons_app') - eq_(answers[0][2], 'extension') + eq_(len(answers), 2) + eq_(answers[0][0], False) + eq_(answers[0][1], 'flask_app') + eq_(answers[1][0], True) + eq_(answers[1][1], 'pylons_app') + eq_(answers[1][2], 'extension') p.unload('test_routing_plugin') @@ -296,14 +290,16 @@ def test_ask_around_pylons_extension_route_post_using_get(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) # We are going to get an answer from Pylons, but just because it will # match the catch-all template route, hence the `core` origin. - eq_(len(answers), 1) - eq_(answers[0][0], True) - eq_(answers[0][1], 'pylons_app') - eq_(answers[0][2], 'core') + eq_(len(answers), 2) + eq_(answers[0][0], False) + eq_(answers[0][1], 'flask_app') + eq_(answers[1][0], True) + eq_(answers[1][1], 'pylons_app') + eq_(answers[1][2], 'core') p.unload('test_routing_plugin') @@ -323,12 +319,14 @@ def test_ask_around_pylons_extension_route_get_after_map(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) - eq_(len(answers), 1) - eq_(answers[0][0], True) - eq_(answers[0][1], 'pylons_app') - eq_(answers[0][2], 'extension') + eq_(len(answers), 2) + eq_(answers[0][0], False) + eq_(answers[0][1], 'flask_app') + eq_(answers[1][0], True) + eq_(answers[1][1], 'pylons_app') + eq_(answers[1][2], 'extension') p.unload('test_routing_plugin') @@ -348,7 +346,7 @@ def test_ask_around_flask_core_and_pylons_extension_route(self): } wsgiref.util.setup_testing_defaults(environ) - answers = app.ask_around('can_handle_request', environ) + answers = app.ask_around(environ) answers = sorted(answers, key=lambda a: a[1]) eq_(len(answers), 2) diff --git a/requirements.in b/requirements.in index 6839d570513..b3ff4ca7fce 100644 --- a/requirements.in +++ b/requirements.in @@ -31,5 +31,4 @@ vdm==0.13 WebHelpers==1.3 WebOb==1.0.8 WebTest==1.4.3 # need to pin this so that Pylons does not install a newer version that conflicts with WebOb==1.0.8 -wsgi-party==0.1b1 zope.interface==4.2.0 diff --git a/requirements.txt b/requirements.txt index db90589524c..7e12c29998a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -53,7 +53,6 @@ WebHelpers==1.3 WebOb==1.0.8 WebTest==1.4.3 Werkzeug==0.11.10 -wsgi-party==0.1b1 zope.interface==4.2.0 # The following packages are commented out because they are From 2df4a09dbeff727dc5a516e25ff9ed154762f5b7 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 1 Jul 2016 16:11:45 +0100 Subject: [PATCH 087/210] [#3148] Remove partyline import --- ckan/tests/config/test_middleware.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 633f92d14f3..60b3b3f147f 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -10,7 +10,7 @@ from ckan.config.middleware import AskAppDispatcherMiddleware from ckan.config.middleware.flask_app import CKANFlask -from ckan.controllers.partyline import PartylineController +from ckan.config.middleware.pylons_app import CKANPylonsApp class TestPylonsResponseCleanupMiddleware(helpers.FunctionalTestBase): @@ -63,7 +63,7 @@ def test_flask_can_handle_request_is_called_with_environ(self): def test_pylons_can_handle_request_is_called_with_environ(self): - with mock.patch.object(PartylineController, 'can_handle_request') as \ + with mock.patch.object(CKANPylonsApp, 'can_handle_request') as \ mock_can_handle_request: # We need set this otherwise the mock object is returned From c73908f86a8cc77de55901455ef885296e5eb13a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20H=C3=BChne?= Date: Mon, 4 Jul 2016 15:50:25 +0200 Subject: [PATCH 088/210] Add information about moved helper to changelog --- CHANGELOG.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 69903015a0c..463c94d73ab 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,11 @@ Changelog --------- +v2.6.0 TBA +================= +API changes and deprecations: + * Replace `c.__version__` with new helper `h.ckan_version()` (#3103) + v2.5.2 2016-03-31 ================= From 0629933a25ac982f87f95743faf001126d3cc2e1 Mon Sep 17 00:00:00 2001 From: Jared Smith Date: Mon, 4 Jul 2016 09:15:16 -0700 Subject: [PATCH 089/210] Added strict comparison and removed whitespace === used for `this.options.field_url` and `this.options.field_upload` in determining `this.is_data_resource` Removed whitespace in variable assignments --- .../base/javascript/modules/image-upload.js | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/ckan/public/base/javascript/modules/image-upload.js b/ckan/public/base/javascript/modules/image-upload.js index 3a622dc051c..c53dba049e6 100644 --- a/ckan/public/base/javascript/modules/image-upload.js +++ b/ckan/public/base/javascript/modules/image-upload.js @@ -40,19 +40,19 @@ this.ckan.module('image-upload', function($, _) { // firstly setup the fields var field_upload = 'input[name="' + options.field_upload + '"]'; - var field_url = 'input[name="' + options.field_url + '"]'; - var field_clear = 'input[name="' + options.field_clear + '"]'; - var field_name = 'input[name="' + options.field_name + '"]'; - - this.input = $(field_upload, this.el); - this.field_url = $(field_url, this.el).parents('.control-group'); - this.field_image = this.input.parents('.control-group'); - this.field_url_input = $('input', this.field_url); - this.field_name = this.el.parents('form').find(field_name); + var field_url = 'input[name="' + options.field_url + '"]'; + var field_clear = 'input[name="' + options.field_clear + '"]'; + var field_name = 'input[name="' + options.field_name + '"]'; + + this.input = $(field_upload, this.el); + this.field_url = $(field_url, this.el).parents('.control-group'); + this.field_image = this.input.parents('.control-group'); + this.field_url_input = $('input', this.field_url); + this.field_name = this.el.parents('form').find(field_name); // this is the location for the upload/link data/image label - this.label_location = $('label[for="field-image-url"]'); + this.label_location = $('label[for="field-image-url"]'); // determines if the resource is a data resource - this.is_data_resource = (this.options.field_url == 'url') && (this.options.field_upload == 'upload'); + this.is_data_resource = (this.options.field_url === 'url') && (this.options.field_upload === 'upload'); // Is there a clear checkbox on the form already? var checkbox = $(field_clear, this.el); From 8177dbc8fd6ee1a4d02e997e7c19b0675c4fce50 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Tue, 5 Jul 2016 15:02:02 +0200 Subject: [PATCH 090/210] Improve JavaScript i18n documentation. Reorganizes the existing documentation in a single place that is linked to from related topics. Also adds more details, for example regarding `paster trans js`. --- doc/contributing/frontend/index.rst | 37 +----- .../frontend/javascript-module-tutorial.rst | 26 ++-- doc/contributing/string-i18n.rst | 114 ++++++++++++++++-- doc/extensions/translating-extensions.rst | 10 ++ .../javascript-module-objects-and-methods.rst | 2 + doc/theming/javascript.rst | 7 +- 6 files changed, 132 insertions(+), 64 deletions(-) diff --git a/doc/contributing/frontend/index.rst b/doc/contributing/frontend/index.rst index 6a70812f2ac..0475f638685 100644 --- a/doc/contributing/frontend/index.rst +++ b/doc/contributing/frontend/index.rst @@ -185,9 +185,8 @@ Modules Modules are the core of the CKAN website, every component that is interactive on the page should be a module. These are then initialized by including a ``data-module`` attribute on an element on the page. For -example: +example:: -:: The idea is to create small isolated components that can easily be @@ -331,39 +330,7 @@ useful in the future. Internationalization ==================== -All strings within modules should be internationalized. Strings can be -set in the ``options.i18n`` object and there is a ``.i18n()`` helper for -retrieving them. - -:: - - ckan.module('language-picker', function (jQuery, _) { - return { - options: { - i18n: { - hello_1: _('Hello'), - hello_2: _('Hello %(name)s'), - apples: function (params) { - var n = params.num; - return _('I have %(num)d apple').isPlural(n, 'I have %(num)d apples'); - } - } - }, - initialize: function () { - // Standard example - this.i18n('hello_1'); // "Hello" - - // String interpolation example - var name = 'Dave'; - this.i18n('hello_2', {name: name}); // "Hello Dave" - - // Plural example - var total = 1; - this.i18n('apples', {num: total}); // "I have 1 apple" - this.i18n('apples', {num: 3}); // "I have 3 apples" - } - } - }); +See :ref:`javascript_i18n`. jQuery plugins diff --git a/doc/contributing/frontend/javascript-module-tutorial.rst b/doc/contributing/frontend/javascript-module-tutorial.rst index 7a732c5e57e..1c793566832 100644 --- a/doc/contributing/frontend/javascript-module-tutorial.rst +++ b/doc/contributing/frontend/javascript-module-tutorial.rst @@ -106,34 +106,30 @@ functionality elsewhere. this.sandbox.client.favoriteDataset(this.button.val()); } -Notifications and Internationalisation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Internationalisation +~~~~~~~~~~~~~~~~~~~~ +See :ref:`javascript_i18n`. + + +Notifications +~~~~~~~~~~~~~ This submits the dataset to the API but ideally we want to tell the user what we're doing. -:: +.. code-block:: javascript - options: { - i18n: { - loading: _('Favouriting dataset'), - done: _('Favourited dataset %(id)s') - } - }, favorite: function () { - // i18n gets a translation key from the options object. - this.button.text(this.i18n('loading')); + this.button.text('Loading'); // The client on the sandbox should always be used to talk to the api. var request = this.sandbox.client.favoriteDataset(this.button.val()) request.done(jQuery.proxy(this._onSuccess, this)); }, _onSuccess: function () { - // We can perform interpolation on messages. - var message = this.i18n('done', {id: this.button.val()}); - // Notify allows global messages to be displayed to the user. - this.sandbox.notify(message, 'success'); + this.sandbox.notify('Done', 'success'); } Options diff --git a/doc/contributing/string-i18n.rst b/doc/contributing/string-i18n.rst index 6909e33c5de..da28699ebe3 100644 --- a/doc/contributing/string-i18n.rst +++ b/doc/contributing/string-i18n.rst @@ -191,12 +191,104 @@ To handle different plural and singular forms of a string, use ``ungettext()``: num_objects).format(count=count, name=name) +.. _javascript_i18n: + --------------------------------------------- Internationalizing strings in JavaScript code --------------------------------------------- -.. todo:: +Each :ref:`CKAN JavaScript module ` receives the ``_`` +translation function during construction: + +.. code-block:: javascript + + this.ckan.module('i18n-demo', function($, _) { // Note the `_` + ... + }; + +Alternatively, you can use :ref:`this.sandbox.translate ` (for +which ``_`` is a shortcut). + +In contrast to Python code and Jinja templates, in Javascript ``_`` does not +directly return the translated string. Instead, it returns a Jed_ translation +object. Use its ``fetch`` method to get the translated singular string: + +.. code-block:: javascript + + _('Translate me!').fetch() + +.. _Jed: http://slexaxton.github.io/Jed/ + +Placeholders can be `specified in the string`_, their values are passed via +``fetch``: + +.. _specified in the string: http://www.diveintojavascript.com/projects/javascript-sprintf + +.. code-block:: javascript + + _('Hello, %(name)s!').fetch({name: 'John Doe'}) + +For plural forms, use the ``ifPlural`` method: + +.. code-block:: javascript + var numDeleted = deletePackages(); + console.log( + _('Deleted %(number)d package') + .ifPlural(numDeleted, 'Deleted %(number)d packages') + .fetch({number: numDeleted}) + ); + +However, you should not distribute your translateable strings all over your +JavaScript module. Instead, collect them in ``options.i18n`` and then use the +``this.i18n`` function to look them up: + +.. code-block:: javascript + + this.ckan.module('i18n-demo', function($, _) { + return { + options: { + i18n: { + deleting _('Deleting...'), + deleted: function (data) { + return _('Deleted %(number)d package') + .isPlural(data.number, 'Deleted %(number)d packages'); + } + } + }, + + // ... + + deleteSomeStuff: function() { + console.log(this.i18n('deleting')); + var numDeleted = deletePackages(); + console.log(this.i18n('deleted', {number: numDeleted})); + } + }; + }; + +As you can see, each entry in ``options.i18n`` maps an identifier to either a +prepared Jed object (as returned by ``_``) or to a function that takes an +object and then returns a Jed object. The latter form is for passing +placeholders and constructing plural forms. Note that you should not call +``fetch`` for the values in ``options.i18n``, that is handled automatically by +``this.i18n``. + +.. note:: + + CKAN's JavaScript code automatically downloads the appropriate translations + at request time from the CKAN server. The corresponding translation files + are generated beforehand using ``paster trans js``. During development you + need to run that command manually if you're updating JavaScript + translations:: + + python setup.py extract_messages # Extract translatable strings + # Update .po files as desired + python setup.py compile_catalog # Compile .mo files for Python/Jinja + paster trans js # Compile JavaScript catalogs + + Also note that extensions currently `cannot provide translations for + JS strings `_. ------------------------------------------------- General guidelines for internationalizing strings @@ -359,20 +451,22 @@ the quality of CKAN's translations: {# TRANSLATORS: This heading is displayed on the user's profile page. #}

{% trans %}Heading{% endtrans %}

+ In JavaScript: + + .. code-block:: javascript + + // TRANSLATORS: "Manual" refers to the user manual + _("Manual") + These comments end up in the ``ckan.pot`` file and translators will see them when they're translating the strings (Transifex shows them, for example). .. note:: - In both Python and Jinja2, the comment must be on the line before the line - with the ``_()``, ``ungettext()`` or ``{% trans %}``, and must start with - the exact string ``TRANSLATORS:`` (in upper-case and with the colon). - This string is configured in ``setup.cfg``. - - .. todo:: - - Example of leaving a translator comment in JavaScript. - Probably ``// TRANSLATORS: This is a helpful comment`` will work. + The comment must be on the line before the line with the ``_()``, + ``ungettext()`` or ``{% trans %}``, and must start with the exact string + ``TRANSLATORS:`` (in upper-case and with the colon). This string is + configured in ``setup.cfg``. .. todo:: diff --git a/doc/extensions/translating-extensions.rst b/doc/extensions/translating-extensions.rst index 43366283ac8..244dfa3027e 100644 --- a/doc/extensions/translating-extensions.rst +++ b/doc/extensions/translating-extensions.rst @@ -163,3 +163,13 @@ implement the ``ITranslation`` interface yourself. ~ckan.plugins.interfaces.ITranslation.i18n_directory ~ckan.plugins.interfaces.ITranslation.i18n_locales ~ckan.plugins.interfaces.ITranslation.i18n_domain + +---------- +JavaScript +---------- +Extensions currently `cannot provide their own translations for JavaScript +strings `_. + +However, they can re-use existing JavaScript translations from CKAN. See +:ref:`javascript_i18n` for details. + diff --git a/doc/theming/javascript-module-objects-and-methods.rst b/doc/theming/javascript-module-objects-and-methods.rst index 7347b1319d6..73660f8553f 100644 --- a/doc/theming/javascript-module-objects-and-methods.rst +++ b/doc/theming/javascript-module-objects-and-methods.rst @@ -23,6 +23,8 @@ module to use, including: document traversal and manipulation, event handling, and animation. See `jQuery's own docs `_ for details. +.. _this_sandbox: + * :js:data:`this.sandbox`, an object containing useful functions for all modules to use, including: diff --git a/doc/theming/javascript.rst b/doc/theming/javascript.rst index 80f6d80865d..a1feaef4c0f 100644 --- a/doc/theming/javascript.rst +++ b/doc/theming/javascript.rst @@ -1,3 +1,5 @@ +.. _javascript_modules: + ============================= Customizing CKAN's JavaScript ============================= @@ -521,10 +523,7 @@ clicked it turns green: Internationalization -------------------- -.. todo:: - - Show how to Internationalize a JavaScript module. - +See :ref:`javascript_i18n`. -------------------------- Testing JavaScript modules From d9a2c6a61e8a234141d54dc7197423b711ebddb2 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 6 Jul 2016 10:54:04 +0100 Subject: [PATCH 091/210] [#3148] Simplify app references and tests --- ckan/config/middleware/flask_app.py | 2 +- ckan/config/middleware/pylons_app.py | 2 +- ckan/tests/config/test_middleware.py | 63 +++++----------------------- 3 files changed, 13 insertions(+), 54 deletions(-) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index f88d576268a..6c96306ce31 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -23,7 +23,7 @@ def hello_world_post(): return 'Hello World, this was posted to Flask' # Add a reference to the actual Flask app so it's easier to access - setattr(app, '_wsgi_app', flask_app) + app._wsgi_app = flask_app return app diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py index 7af95aec9c2..7158e478f4f 100644 --- a/ckan/config/middleware/pylons_app.py +++ b/ckan/config/middleware/pylons_app.py @@ -179,7 +179,7 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): app = common_middleware.RootPathMiddleware(app, config) # Add a reference to the actual Pylons app so it's easier to access - setattr(app, '_wsgi_app', pylons_app) + app._wsgi_app = pylons_app return app diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 60b3b3f147f..fad66fa4b6d 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -148,10 +148,9 @@ def test_ask_around_flask_core_route_get(self): # Even though this route is defined in Flask, there is catch all route # in Pylons for all requests to point arbitrary urls to templates with # the same name, so we get two positive answers - eq_(len(answers), 2) - eq_([a[0] for a in answers], [True, True]) - eq_(sorted([a[1] for a in answers]), ['flask_app', 'pylons_app']) - # TODO: check origin (core/extension) when that is in place + eq_(answers, [(True, 'flask_app'), (True, 'pylons_app', 'core')]) + # TODO: check Flask origin (core/extension) when that is in place + # (also on the following tests) def test_ask_around_flask_core_route_post(self): @@ -171,10 +170,7 @@ def test_ask_around_flask_core_route_post(self): # Even though this route is defined in Flask, there is catch all route # in Pylons for all requests to point arbitrary urls to templates with # the same name, so we get two positive answers - eq_(len(answers), 2) - eq_([a[0] for a in answers], [True, True]) - eq_(sorted([a[1] for a in answers]), ['flask_app', 'pylons_app']) - # TODO: check origin (core/extension) when that is in place + eq_(answers, [(True, 'flask_app'), (True, 'pylons_app', 'core')]) def test_ask_around_pylons_core_route_get(self): @@ -191,12 +187,7 @@ def test_ask_around_pylons_core_route_get(self): answers = app.ask_around(environ) - eq_(len(answers), 2) - eq_(answers[0][0], False) - eq_(answers[0][1], 'flask_app') - eq_(answers[1][0], True) - eq_(answers[1][1], 'pylons_app') - eq_(answers[1][2], 'core') + eq_(answers, [(False, 'flask_app'), (True, 'pylons_app', 'core')]) def test_ask_around_pylons_core_route_post(self): @@ -213,12 +204,7 @@ def test_ask_around_pylons_core_route_post(self): answers = app.ask_around(environ) - eq_(len(answers), 2) - eq_(answers[0][0], False) - eq_(answers[0][1], 'flask_app') - eq_(answers[1][0], True) - eq_(answers[1][1], 'pylons_app') - eq_(answers[1][2], 'core') + eq_(answers, [(False, 'flask_app'), (True, 'pylons_app', 'core')]) def test_ask_around_pylons_extension_route_get_before_map(self): @@ -238,12 +224,7 @@ def test_ask_around_pylons_extension_route_get_before_map(self): answers = app.ask_around(environ) - eq_(len(answers), 2) - eq_(answers[0][0], False) - eq_(answers[0][1], 'flask_app') - eq_(answers[1][0], True) - eq_(answers[1][1], 'pylons_app') - eq_(answers[1][2], 'extension') + eq_(answers, [(False, 'flask_app'), (True, 'pylons_app', 'extension')]) p.unload('test_routing_plugin') @@ -265,12 +246,7 @@ def test_ask_around_pylons_extension_route_post(self): answers = app.ask_around(environ) - eq_(len(answers), 2) - eq_(answers[0][0], False) - eq_(answers[0][1], 'flask_app') - eq_(answers[1][0], True) - eq_(answers[1][1], 'pylons_app') - eq_(answers[1][2], 'extension') + eq_(answers, [(False, 'flask_app'), (True, 'pylons_app', 'extension')]) p.unload('test_routing_plugin') @@ -294,12 +270,7 @@ def test_ask_around_pylons_extension_route_post_using_get(self): # We are going to get an answer from Pylons, but just because it will # match the catch-all template route, hence the `core` origin. - eq_(len(answers), 2) - eq_(answers[0][0], False) - eq_(answers[0][1], 'flask_app') - eq_(answers[1][0], True) - eq_(answers[1][1], 'pylons_app') - eq_(answers[1][2], 'core') + eq_(answers, [(False, 'flask_app'), (True, 'pylons_app', 'core')]) p.unload('test_routing_plugin') @@ -321,12 +292,7 @@ def test_ask_around_pylons_extension_route_get_after_map(self): answers = app.ask_around(environ) - eq_(len(answers), 2) - eq_(answers[0][0], False) - eq_(answers[0][1], 'flask_app') - eq_(answers[1][0], True) - eq_(answers[1][1], 'pylons_app') - eq_(answers[1][2], 'extension') + eq_(answers, [(False, 'flask_app'), (True, 'pylons_app', 'extension')]) p.unload('test_routing_plugin') @@ -349,14 +315,7 @@ def test_ask_around_flask_core_and_pylons_extension_route(self): answers = app.ask_around(environ) answers = sorted(answers, key=lambda a: a[1]) - eq_(len(answers), 2) - eq_([a[0] for a in answers], [True, True]) - eq_([a[1] for a in answers], ['flask_app', 'pylons_app']) - - # TODO: we still can't distinguish between Flask core and extension - # eq_(answers[0][2], 'extension') - - eq_(answers[1][2], 'extension') + eq_(answers, [(True, 'flask_app'), (True, 'pylons_app', 'extension')]) p.unload('test_routing_plugin') From 9f635844a2c69f70f33004488622f40cd39f055f Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Wed, 6 Jul 2016 15:51:35 +0200 Subject: [PATCH 092/210] Make sure that `ckan.lib.munge.munge_filename` returns Unicode. --- ckan/lib/munge.py | 40 +++++++++++++++++++++++++----------- ckan/tests/lib/test_munge.py | 4 +++- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/ckan/lib/munge.py b/ckan/lib/munge.py index 2f501adc422..7570b588c39 100644 --- a/ckan/lib/munge.py +++ b/ckan/lib/munge.py @@ -9,6 +9,17 @@ import os.path from ckan import model +from ckan.lib.io import decode_path + + +# Maximum length of a filename's extension (including the '.') +MAX_FILENAME_EXTENSION_LENGTH = 21 + +# Maximum total length of a filename (including extension) +MAX_FILENAME_TOTAL_LENGTH = 100 + +# Minimum total length of a filename (including extension) +MIN_FILENAME_TOTAL_LENGTH = 3 def munge_name(name): @@ -134,22 +145,27 @@ def munge_filename(filename): Keeps the filename extension (e.g. .csv). Strips off any path on the front. + + Returns a Unicode string. ''' + if not isinstance(filename, unicode): + filename = decode_path(filename) - # just get the filename ignore the path - path, filename = os.path.split(filename) - # clean up - filename = substitute_ascii_equivalents(filename) + # Ignore path + filename = os.path.split(filename)[1] + + # Clean up filename = filename.lower().strip() - filename = re.sub(r'[^a-zA-Z0-9_. -]', '', filename).replace(' ', '-') - # resize if needed but keep extension + filename = substitute_ascii_equivalents(filename) + filename = re.sub(ur'[^a-zA-Z0-9_. -]', '', filename).replace(u' ', u'-') + filename = re.sub(ur'-+', u'-', filename) + + # Enforce length constraints name, ext = os.path.splitext(filename) - # limit overly long extensions - if len(ext) > 21: - ext = ext[:21] - # max/min size - ext_length = len(ext) - name = _munge_to_length(name, max(3 - ext_length, 1), 100 - ext_length) + ext = ext[:MAX_FILENAME_EXTENSION_LENGTH] + ext_len = len(ext) + name = _munge_to_length(name, max(1, MIN_FILENAME_TOTAL_LENGTH - ext_len), + MAX_FILENAME_TOTAL_LENGTH - ext_len) filename = name + ext return filename diff --git a/ckan/tests/lib/test_munge.py b/ckan/tests/lib/test_munge.py index 50225ed7e68..0f89fc3096a 100644 --- a/ckan/tests/lib/test_munge.py +++ b/ckan/tests/lib/test_munge.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from nose.tools import assert_equal +from nose.tools import assert_equal, ok_ from ckan.lib.munge import (munge_filename_legacy, munge_filename, munge_name, munge_title_to_name, munge_tag) @@ -57,6 +57,7 @@ class TestMungeFilename(object): ('path/to/file.csv', 'file.csv'), ('.longextension', '.longextension'), ('a.longextension', 'a.longextension'), + ('a.now_that_extension_is_too_long', 'a.now_that_extension_i'), ('.1', '.1_'), ] @@ -65,6 +66,7 @@ def test_munge_filename(self): for org, exp in self.munge_list: munge = munge_filename(org) assert_equal(munge, exp) + ok_(isinstance(munge, unicode)) def test_munge_filename_multiple_pass(self): '''Munging filename multiple times produces same result.''' From 9ba0ecf89b05a626f7068cbdb94d1465225259a0 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Wed, 6 Jul 2016 15:52:15 +0200 Subject: [PATCH 093/210] Documentation and tools for Unicode filename handling. Documents best practices for dealing with Unicode filenames. Also adds the module `ckan.lib.io` which provides `decode_path` and `encode_path` for de-/encoding Unicode filenames. --- ckan/lib/io.py | 52 +++++++++++++++++++++++ ckan/tests/lib/test_io.py | 41 ++++++++++++++++++ doc/contributing/unicode.rst | 82 +++++++++++++++++++++++++++++++++++- 3 files changed, 174 insertions(+), 1 deletion(-) create mode 100644 ckan/lib/io.py create mode 100644 ckan/tests/lib/test_io.py diff --git a/ckan/lib/io.py b/ckan/lib/io.py new file mode 100644 index 00000000000..edafe8fc549 --- /dev/null +++ b/ckan/lib/io.py @@ -0,0 +1,52 @@ +# encoding: utf-8 + +u''' +Utility functions for I/O. +''' + +import sys + + +_FILESYSTEM_ENCODING = unicode(sys.getfilesystemencoding() + or sys.getdefaultencoding()) + + +def encode_path(p): + u''' + Convert a Unicode path string to a byte string. + + Intended to be used for encoding paths that are known to be + compatible with the filesystem, for example paths of existing files + that were previously decoded using :py:func:`decode_path`. If you're + dynamically constructing names for new files using unknown inputs + then pass them through :py:func:`ckan.lib.munge.munge_filename` + before encoding them. + + Raises a ``UnicodeEncodeError`` if the path cannot be encoded using + the filesystem's encoding. That will never happen for paths returned + by :py:func:`decode_path`. + + Raises a ``TypeError`` is the input is not a Unicode string. + ''' + if not isinstance(p, unicode): + raise TypeError(u'Can only encode unicode, not {}'.format(type(p))) + return p.encode(_FILESYSTEM_ENCODING) + + +def decode_path(p): + u''' + Convert a byte path string to a Unicode string. + + Intended to be used for decoding byte paths to existing files as + returned by some of Python's built-in I/O functions. + + Raises a ``UnicodeDecodeError`` if the path cannot be decoded using + the filesystem's encoding. Assuming the path was returned by one of + Python's I/O functions this means that the environment Python is + running in is set up incorrectly. + + Raises a ``TypeError`` if the input is not a byte string. + ''' + if not isinstance(p, str): + raise TypeError(u'Can only decode str, not {}'.format(type(p))) + return p.decode(_FILESYSTEM_ENCODING) diff --git a/ckan/tests/lib/test_io.py b/ckan/tests/lib/test_io.py new file mode 100644 index 00000000000..13e06a37aab --- /dev/null +++ b/ckan/tests/lib/test_io.py @@ -0,0 +1,41 @@ +# encoding: utf-8 + +import io +import os.path +import shutil +import tempfile + +from nose.tools import eq_, ok_, raises + +import ckan.lib.io as ckan_io + + +class TestDecodeEncodePath(object): + + @raises(TypeError) + def test_decode_path_fails_for_unicode(self): + ckan_io.decode_path(u'just_a_unicode') + + @raises(TypeError) + def test_encode_path_fails_for_str(self): + ckan_io.encode_path(b'just_a_str') + + def test_decode_path_returns_unicode(self): + ok_(isinstance(ckan_io.decode_path(b'just_a_str'), unicode)) + + def test_encode_path_returns_str(self): + ok_(isinstance(ckan_io.encode_path(u'just_a_unicode'), str)) + + def test_decode_encode_path(self): + temp_dir = ckan_io.decode_path(tempfile.mkdtemp()) + try: + filename = u'\xf6\xe4\xfc.txt' + path = os.path.join(temp_dir, filename) + with io.open(ckan_io.encode_path(path), u'w', + encoding=u'utf-8') as f: + f.write(u'foo') + # Force str return type + filenames = os.listdir(ckan_io.encode_path(temp_dir)) + eq_(ckan_io.decode_path(filenames[0]), filename) + finally: + shutil.rmtree(temp_dir) diff --git a/doc/contributing/unicode.rst b/doc/contributing/unicode.rst index eb4a803d72b..98baddfa07c 100644 --- a/doc/contributing/unicode.rst +++ b/doc/contributing/unicode.rst @@ -83,7 +83,7 @@ When opening text (not binary) files you should use `io.open`_ instead of import io - with io.open(b'my_file.txt', u'r', encoding=u'utf-8') as f: + with io.open(u'my_file.txt', u'r', encoding=u'utf-8') as f: text = f.read() # contents is automatically decoded # to unicode using UTF-8 @@ -144,6 +144,14 @@ flag:: >>> print re.match(ur'^\w$', u'ö', re.U) <_sre.SRE_Match object at 0xb60ea2f8> +.. note:: + + Some functions (e.g. ``re.split`` and ``re.sub``) take additional optional + parameters before the flags, so you should pass the flag via a keyword + argument:: + + replaced = re.sub(ur'\W', u'_', original, flags=re.U) + The type of the values returned by ``re.split``, ``re.MatchObject.group``, etc. depends on the type of the input string:: @@ -158,3 +166,75 @@ Note that the type of the *pattern string* does not influence the return type. .. _re: https://docs.python.org/2/library/re.html .. _re.U: https://docs.python.org/2/library/re.html#re.U + +Filenames +````````` +Like all other strings, filenames should be stored as Unicode strings +internally. However, some filesystem operations return or expect byte strings, +so filenames have to be encoded/decoded appropriately. Unfortunately, different +operating systems use different encodings for their filenames, and on some of +them (e.g. Linux) the file system encoding is even configurable by the user. + +To make decoding and encoding of filenames easier, the ``ckan.lib.io`` module +therefore contains the functions ``decode_path`` and ``encode_path``, which +automatically use the correct encoding:: + + import io + import json + + from ckan.lib.io import decode_path + + # __file__ is a byte string, so we decode it + MODULE_FILE = decode_path(__file__) + print(u'Running from ' + MODULE_FILE) + + # The functions in os.path return unicode if given unicode + MODULE_DIR = os.path.dirname(MODULE_FILE) + DATA_FILE = os.path.join(MODULE_DIR, u'data.json') + + # Most of Python's built-in I/O-functions accept Unicode filenames as input + # and encode them automatically + with io.open(DATA_FILE, encoding='utf-8') as f: + data = json.load(f) + +Note that almost all Python's built-in I/O-functions accept Unicode filenames +as input and encode them automatically, so using ``encode_path`` is usually not +necessary. + +The return type of some of Python's I/O-functions (e.g. os.listdir_ and +os.walk_) depends on the type of their input: If passed byte strings they +return byte strings and if passed Unicode they automatically decode the raw +filenames to Unicode before returning them. Other functions exist in two +variants that return byte strings (e.g. os.getcwd_) and Unicode (os.getcwdu_), +respectively. + +.. warning:: + + Some of Python's I/O-functions may return *both* byte and Unicode strings + for *a single* call. For example, os.listdir_ will normally return Unicode + when passed Unicode, but filenames that cannot be decoded using the + filesystem encoding will still be returned as byte strings! + + Note that if the filename of an existing file cannot be decoded using the + filesystem's encoding then the environment Python is running in is most + probably incorrectly set up. + +The instructions above are meant for the names of existing files that are +obtained using Python's I/O functions. However, sometimes one also wants to +create new files whose names are generated from unknown sources (e.g. user +input). To make sure that the generated filename is safe to use and can be +represented using the filesystem's encoding use +``ckan.lib.munge.munge_filename``:: + + >> ckan.lib.munge.munge_filename(u'Data from Linköping (year: 2016).txt') + u'data-from-linkoping-year-2016.txt' + +.. note:: + + ``munge_filename`` will remove a leading path from the filename. + +.. _os.listdir: https://docs.python.org/2/library/os.html#os.listdir +.. _os.walk: https://docs.python.org/2/library/os.html#os.walk +.. _os.getcwd: https://docs.python.org/2/library/os.html#os.getcwd +.. _os.getcwdu: https://docs.python.org/2/library/os.html#os.getcwdu + From 4b56e2344c180b4c16150e1e20df30afcb4796b7 Mon Sep 17 00:00:00 2001 From: Florian Brucker Date: Fri, 8 Jul 2016 15:05:27 +0200 Subject: [PATCH 094/210] [#2952] Depcreate `db load` and `db dump` Deprecates the paster commands `db load` and `db dump` in favor of PostgreSQL's `pg_dump` and `pg_restore`. `db load` and `db dump` are kept but print a warning, and their documentation is replaced with a documentation of how to dump/import the database using `pg_dump` and `pg_restore`. --- ckan/lib/cli.py | 20 +- doc/contributing/database-migrations.rst | 5 +- doc/extensions/tutorial.rst | 3 +- doc/maintaining/configuration.rst | 2 +- doc/maintaining/database-management.rst | 171 ++++++++++++++++++ doc/maintaining/getting-started.rst | 2 +- doc/maintaining/index.rst | 1 + .../installing/install-from-source.rst | 2 +- doc/maintaining/paster.rst | 111 +----------- doc/maintaining/upgrading/index.rst | 12 +- .../upgrading/upgrade-package-ckan-1-to-2.rst | 16 +- .../upgrade-package-to-minor-release.rst | 19 +- doc/maintaining/upgrading/upgrade-source.rst | 20 +- 13 files changed, 207 insertions(+), 177 deletions(-) create mode 100644 doc/maintaining/database-management.rst diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 86537863c3b..7014029670b 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -31,6 +31,18 @@ # Otherwise loggers get disabled. +def deprecation_warning(message=None): + ''' + Print a deprecation warning to STDERR. + + If ``message`` is given it is also printed to STDERR. + ''' + sys.stderr.write(u'WARNING: This function is deprecated.') + if message: + sys.stderr.write(u' ' + message.strip()) + sys.stderr.write(u'\n') + + def parse_db_config(config_key='sqlalchemy.url'): ''' Takes a config key for a database connection url and parses it into a dictionary. Expects a url like: @@ -187,10 +199,10 @@ class ManageDb(CkanCommand): search index db upgrade [version no.] - Data migrate db version - returns current version of data schema - db dump FILE_PATH - dump to a pg_dump file - db load FILE_PATH - load a pg_dump from a file + db dump FILE_PATH - dump to a pg_dump file [DEPRECATED] + db load FILE_PATH - load a pg_dump from a file [DEPRECATED] db load-only FILE_PATH - load a pg_dump from a file but don\'t do - the schema upgrade or search indexing + the schema upgrade or search indexing [DEPRECATED] db create-from-model - create database from the model (indexes not made) db migrate-filestore - migrate all uploaded data from the 2.1 filesore. ''' @@ -291,6 +303,7 @@ def _run_cmd(self, command_line): raise SystemError('Command exited with errorcode: %i' % retcode) def dump(self): + deprecation_warning(u"Use PostgreSQL's pg_dump instead.") if len(self.args) < 2: print 'Need pg_dump filepath' return @@ -300,6 +313,7 @@ def dump(self): pg_cmd = self._postgres_dump(dump_path) def load(self, only_load=False): + deprecation_warning(u"Use PostgreSQL's pg_restore instead.") if len(self.args) < 2: print 'Need pg_dump filepath' return diff --git a/doc/contributing/database-migrations.rst b/doc/contributing/database-migrations.rst index 9804cc7dbca..2cae6253cf3 100644 --- a/doc/contributing/database-migrations.rst +++ b/doc/contributing/database-migrations.rst @@ -8,9 +8,8 @@ databases to the new database schema when they upgrade their copies of CKAN. These migration scripts are kept in ``ckan.migration.versions``. When you upgrade a CKAN instance, as part of the upgrade process you run any -necessary migration scripts with the ``paster db upgrade`` command:: - - paster --plugin=ckan db upgrade --config={.ini file} +necessary migration scripts with the :ref:`paster db upgrade ` +command. A migration script should be checked into CKAN at the same time as the model changes it is related to. Before pushing the changes, ensure the tests pass diff --git a/doc/extensions/tutorial.rst b/doc/extensions/tutorial.rst index f0e4bf4ea13..2a5a88ce2e2 100644 --- a/doc/extensions/tutorial.rst +++ b/doc/extensions/tutorial.rst @@ -414,7 +414,8 @@ group named ``curators``. If you've already created a ``curators`` group and want to test what happens when the site has no ``curators`` group, you can use CKAN's command line - interface to :ref:`clean and reinitialize your database `. + interface to :ref`clean and reinitialize your database + `. Try visiting the ``/group`` page in CKAN with our ``example_iauthfunctions`` plugin activated in your CKAN config file and with no ``curators`` group in diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index 67e1521f56b..5ab5824d70b 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -992,7 +992,7 @@ web interface. For example:: ckan.dumps_url = http://ckan.net/dump/ -For more information on using dumpfiles, see :ref:`paster db`. +For more information on using dumpfiles, see :ref:`datasets dump`. .. _ckan.dumps_format: diff --git a/doc/maintaining/database-management.rst b/doc/maintaining/database-management.rst new file mode 100644 index 00000000000..74cc49282c4 --- /dev/null +++ b/doc/maintaining/database-management.rst @@ -0,0 +1,171 @@ +.. _database management: + +=================== +Database Management +=================== + +.. note:: + + See :doc:`paster` for details on running the ``paster`` commands + mentioned below. + + +.. _db init: + +Initialization +-------------- + +Before you can run CKAN for the first time, you need to run ``db init`` to +initialize your database: + +.. parsed-literal:: + + paster db init -c |production.ini| + +If you forget to do this you'll see this error message in your web browser: + + 503 Service Unavailable: This site is currently off-line. Database is not + initialised. + + +.. _db clean: + +Cleaning +-------- + +.. warning:: + + This will delete all data from your CKAN database! + +You can delete everything in the CKAN database, including the tables, to start +from scratch: + +.. parsed-literal:: + + paster db clean -c |production.ini| + +After cleaning the database you must do either `initialize it`_ or `import +a previously created dump`_. + +.. _initialize it: Initialization_ +.. _import a previously created dump: `db dumping and loading`_ + + +Import and Export +----------------- + +.. _db dumping and loading: + +Dumping and Loading databases to/from a file +```````````````````````````````````````````` + +PostgreSQL offers the command line tools pg_dump_ and pg_restore_ for dumping +and restoring a database and its content to/from a file. + +For example, first dump your CKAN database:: + + sudo -u postgres pg_dump --format=custom -d ckan_default > ckan.dump + +.. warning:: + + The exported file is a complete backup of the database, and includes API + keys and other user data which may be regarded as private. So keep it + secure, like your database server. + +.. note:: + + If you've chosen a non-default database name (i.e. *not* ``ckan_default``) + then you need to adapt the commands accordingly. + +Then restore it again: + +.. parsed-literal:: + + paster db clean -c |production.ini| + sudo -u postgres pg_restore --clean --if-exists -d ckan_default < ckan.dump + +If you're importing a dump from an older version of CKAN you must :ref:`upgrade +the database schema ` after the import. + +Once the import (and a potential upgrade) is complete you should :ref:`rebuild +the search index `. + +.. note:: + + Earlier versions of CKAN offered the ``paster`` commands ``db dump`` and + ``db load``. These are still available but are deprecated in favor of + the native tools of PostgreSQL mentioned above. ``db dump`` and ``db load`` + will be removed in future versions of CKAN. + +.. _pg_dump: https://www.postgresql.org/docs/current/static/app-pgdump.html +.. _pg_restore: https://www.postgresql.org/docs/current/static/app-pgrestore.html + + +.. _datasets dump: + +Exporting Datasets to JSON Lines +```````````````````````````````` + +You can export all of your CKAN site's datasets from your database to a JSON +Lines file using ckanapi_: + +.. parsed-literal:: + + ckanapi dump datasets -c |production.ini| -O my_datasets.jsonl + +This is useful to create a simple public listing of the datasets, with no user +information. Some simple additions to the Apache config can serve the dump +files to users in a directory listing. To do this, add these lines to your +virtual Apache config file (e.g. |apache_config_file|):: + + Alias /dump/ /home/okfn/var/srvc/ckan.net/dumps/ + + # Disable the mod_python handler for static files + + SetHandler None + Options +Indexes + + +.. warning:: + + Don't serve an SQL dump of your database (created using the ``pg_dump`` + command), as those contain private user information such as email + addresses and API keys. + +.. _ckanapi: https://github.com/ckan/ckanapi + + +.. _users dump: + +Exporting User Accounts to JSON Lines +````````````````````````````````````` + +You can export all of your CKAN site's user accounts from your database to +a JSON Lines file using ckanapi_: + +.. parsed-literal:: + + ckanapi dump users -c |production.ini| -O my_database_users.jsonl + + +.. _db upgrade: + +Upgrading +--------- + +.. warning:: + + You should :ref:`create a backup of your database ` + before upgrading it. + + To avoid problems during the database upgrade, comment out any plugins + that you have enabled in your ini file. You can uncomment them again when + the upgrade finishes. + +If you are upgrading to a new CKAN :ref:`major release ` update your +CKAN database's schema using the ``paster db upgrade`` command: + +.. parsed-literal:: + + paster db upgrade -c |production.ini| + diff --git a/doc/maintaining/getting-started.rst b/doc/maintaining/getting-started.rst index e4d05562512..38bc63509e7 100644 --- a/doc/maintaining/getting-started.rst +++ b/doc/maintaining/getting-started.rst @@ -64,7 +64,7 @@ command line with the ``create-test-data`` command: paster create-test-data -c |production.ini| If you later want to delete this test data and start again with an empty -database, you can use the ``db clean`` command, see :ref:`paster db`. +database, you can use the :ref:`db clean ` command. For a list of other command line commands for creating tests data, run:: diff --git a/doc/maintaining/index.rst b/doc/maintaining/index.rst index 57dd7db4364..01763eee6c9 100644 --- a/doc/maintaining/index.rst +++ b/doc/maintaining/index.rst @@ -11,6 +11,7 @@ installing, upgrading and configuring CKAN and its features and extensions. installing/index upgrading/index getting-started + database-management paster authorization data-viewer diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index f5f895333cf..4bcc247ca26 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -325,7 +325,7 @@ installed, we need to install and configure Solr. ------------------------- Now that you have a configuration file that has the correct settings for your -database, you can create the database tables: +database, you can :ref:`create the database tables `: .. parsed-literal:: diff --git a/doc/maintaining/paster.rst b/doc/maintaining/paster.rst index d7490ad2da3..3ec72316443 100644 --- a/doc/maintaining/paster.rst +++ b/doc/maintaining/paster.rst @@ -254,120 +254,11 @@ Usage:: datastore set-permissions - shows a SQL script to execute -.. _paster db: - db: Manage databases ==================== -Lets you initialise, upgrade, and dump the CKAN database. - -Initialization --------------- - -Before you can run CKAN for the first time, you need to run ``db init`` to -initialize your database: - -.. parsed-literal:: - - paster db init -c |production.ini| - -If you forget to do this you'll see this error message in your web browser: - - 503 Service Unavailable: This site is currently off-line. Database is not - initialised. - -Cleaning --------- - -You can delete everything in the CKAN database, including the tables, to start -from scratch: - -.. warning:: - - This will delete all data from your CKAN database! - -.. parsed-literal:: - - paster db clean -c |production.ini| - -After cleaning the db you must do a ``db init`` or ``db load`` before CKAN will -work again. - -.. _dumping and loading: - -Dumping and Loading databases to/from a file --------------------------------------------- - -You can 'dump' (save) the exact state of the database to a file on disk and at -a later point 'load' (restore) it again. - -.. tip:: - - You can also dump the database from one CKAN instance, and then load it into - another CKAN instance on the same or another machine. This will even work if - the CKAN instance you dumped the database from is an older version of CKAN - than the one you load it into, the database will be automatically upgraded - during the load command. (But you cannot load a database from a newer - version of CKAN into an older version of CKAN.) - -To export a dump of your CKAN database: - -.. parsed-literal:: - - paster db dump -c |production.ini| my_database_dump.sql - -To load it in again, you first have to clean the database (this will delete all -data in the database!) and then load the file: - -.. parsed-literal:: - - paster db clean -c |production.ini| - paster db load -c |production.ini| my_database_dump.sql - -.. warning: - - The exported file is a complete backup of the database in plain text, and - includes API keys and other user data which may be regarded as private. So - keep it secure, like your database server. - -Exporting Datasets to JSON Lines --------------------------------- - -You can export all of your CKAN site's datasets from your database to a JSON -Lines file using the ``ckanapi dump datasets`` command: - -.. parsed-literal:: - - ckanapi dump datasets -c |production.ini| -O my_datasets.jsonl - -This is useful to create a simple public listing of the datasets, with no user -information. Some simple additions to the Apache config can serve the dump -files to users in a directory listing. To do this, add these lines to your -virtual Apache config file (e.g. |apache_config_file|):: - - Alias /dump/ /home/okfn/var/srvc/ckan.net/dumps/ - - # Disable the mod_python handler for static files - - SetHandler None - Options +Indexes - - -.. warning:: - - Don't serve an SQL dump of your database (created using the ``paster db - dump`` command), as those contain private user information such as email - addresses and API keys. - -Exporting User Accounts to JSON Lines -------------------------------------- - -You can export all of your CKAN site's user accounts from your database to -a JSON Lines file using the ``ckanapi dump users`` command: - -.. parsed-literal:: +See :doc:`database-management`. - ckanapi dump users -c |production.ini| -O my_database_users.jsonl front-end-build: Creates and minifies css and JavaScript files ============================================================== diff --git a/doc/maintaining/upgrading/index.rst b/doc/maintaining/upgrading/index.rst index 751258614a7..d888ec2f3f5 100644 --- a/doc/maintaining/upgrading/index.rst +++ b/doc/maintaining/upgrading/index.rst @@ -109,17 +109,7 @@ to its pre-upgrade state. |activate| cd |virtualenv|/src/ckan -#. Backup your CKAN database using the ``db dump`` command, for - example: - - .. parsed-literal:: - - paster db dump --config=\ |development.ini| my_ckan_database.pg_dump - - This will create a file called ``my_ckan_database.pg_dump``, you can use the - the ``db load`` command to restore your database to the state recorded in - this file. See :ref:`paster db` for details of the ``db dump`` and ``db - load`` commands. +#. :ref:`Backup your CKAN database ` 2. Upgrade CKAN diff --git a/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst b/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst index 846a3d61998..82e514e1ead 100644 --- a/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst +++ b/doc/maintaining/upgrading/upgrade-package-ckan-1-to-2.rst @@ -19,9 +19,7 @@ your CKAN 1.x site, or on a different machine) and then manually migrate your database and any custom configuration, extensions or templates to your new CKAN 2.x site. We will outline the main steps for migrating below. -#. Create a dump of your CKAN 1.x database:: - - sudo -u ckanstd /var/lib/ckan/std/pyenv/bin/paster --plugin=ckan db dump db-1.x.dump --config=/etc/ckan/std/std.ini +#. :ref:`Create a dump ` of your CKAN 1.x database. #. If you want to install CKAN 2.x on the same server that your CKAN 1.x site was on, uninstall the CKAN 1.x package first:: @@ -46,15 +44,9 @@ database and any custom configuration, extensions or templates to your new CKAN |activate| cd |virtualenv|/src/ckan - Now, load your database. **This will delete any data already present in your - new CKAN 2.x database**. If you've installed CKAN 2.x on a different - machine from 1.x, first copy the database dump file to that machine. - Then run these commands: - - .. parsed-literal:: - - paster db clean -c |production.ini| - paster db load -c |production.ini| db-1.x.dump + Now :ref:`load your database dump ` into CKAN 2.x. + If you've installed CKAN 2.x on a different machine from 1.x, first copy the + database dump file to that machine. #. If you had any custom config settings in your CKAN 1.x instance that you want to copy across to your CKAN 2.x instance, then update your CKAN 2.x diff --git a/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst b/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst index 379f83f70bc..24a4a922d39 100644 --- a/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst +++ b/doc/maintaining/upgrading/upgrade-package-to-minor-release.rst @@ -55,23 +55,8 @@ respectively. you will need to manually reinstall it. #. If there have been changes in the database schema (check the - :doc:`/changelog` to find out) you need to update your CKAN database's - schema using the ``db upgrade`` command. - - .. warning :: - - To avoid problems during the database upgrade, comment out any plugins - that you have enabled in your ini file. You can uncomment them again when - the upgrade finishes. - - For example: - - .. parsed-literal:: - - paster db upgrade --config=\ |development.ini| - - See :ref:`paster db` for details of the ``db upgrade`` - command. + :doc:`/changelog` to find out) you need to :ref:`upgrade your database + schema `. #. If there have been changes in the Solr schema (check the :doc:`/changelog` to find out) you need to restart Jetty for the changes to take effect: diff --git a/doc/maintaining/upgrading/upgrade-source.rst b/doc/maintaining/upgrading/upgrade-source.rst index b8c487f7956..4e89ff313d2 100644 --- a/doc/maintaining/upgrading/upgrade-source.rst +++ b/doc/maintaining/upgrading/upgrade-source.rst @@ -50,23 +50,9 @@ CKAN release you're upgrading to: sudo service jetty restart -#. If you are upgrading to a new :ref:`major release ` update your - CKAN database's schema using the ``db upgrade`` command. - - .. warning :: - - To avoid problems during the database upgrade, comment out any plugins - that you have enabled in your ini file. You can uncomment them again when - the upgrade finishes. - - For example: - - .. parsed-literal:: - - paster db upgrade --config=\ |development.ini| - - See :ref:`paster db` for details of the ``db upgrade`` - command. +#. If there have been changes in the database schema (check the + :doc:`/changelog` to find out) you need to :ref:`upgrade your database + schema `. #. Rebuild your search index by running the ``ckan search-index rebuild`` command: From 601eec2726cec5b9589c19c37dfa610c89d9ec2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Barroso=20Victor?= Date: Mon, 11 Jul 2016 14:52:21 -0300 Subject: [PATCH 095/210] Defining 'count' for search_form snippet Parameter 'count' was not defined as an input for search_form snippet. The template bulk_process was getting an UndefinedError from within the snippet. I included this parameter and the error was fixed. --- ckan/templates/organization/bulk_process.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/organization/bulk_process.html b/ckan/templates/organization/bulk_process.html index 6bdbe039eac..dcb335071eb 100644 --- a/ckan/templates/organization/bulk_process.html +++ b/ckan/templates/organization/bulk_process.html @@ -98,7 +98,7 @@

(_('Name Descending'), 'title_string desc'), (_('Last Modified'), 'data_modified desc') ] %} - {% snippet 'snippets/search_form.html', form_id='organization-datasets-search-form', type='dataset', query=c.q, sorting=sorting, sorting_selected=c.sort_by_selected, no_title=true, search_class=' ' %} + {% snippet 'snippets/search_form.html', form_id='organization-datasets-search-form', type='dataset', query=c.q, count=c.page.item_count, sorting=sorting, sorting_selected=c.sort_by_selected, no_title=true, search_class=' ' %} {% endblock %} {#{% snippet 'snippets/simple_search.html', q=c.q, sort=c.sort_by_selected, placeholder=_('Search datasets...'), extra_sort=[(_('Last Modified'), 'data_modified asc')], input_class='search-normal', form_class='search-aside' %}#} From 048632d73588ed462a1133e796681056fe89ef37 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 23 Jun 2016 11:34:45 +0100 Subject: [PATCH 096/210] [#2842] Refactor evironment loading, initialize Flask configuration Call `load_environment` before making both stacks rather than from the Pylons one. Call the app_globals code on the common environment code, not just on the Pylons stack. Update the Flask config object with all the CKAN values. Don't pass explicitly the configuration object to the `load_all` plugins function, use the one in common. --- ckan/config/environment.py | 16 +++++++++------- ckan/config/middleware/__init__.py | 5 ++++- ckan/config/middleware/flask_app.py | 6 +++++- ckan/config/middleware/pylons_app.py | 21 +++++++++------------ ckan/lib/app_globals.py | 4 +++- ckan/plugins/core.py | 5 ++++- ckan/tests/legacy/test_plugins.py | 10 +++++----- 7 files changed, 39 insertions(+), 28 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index bfc07556376..d668a11b87f 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -1,6 +1,6 @@ # encoding: utf-8 -"""Pylons environment configuration""" +'''CKAN environment configuration''' import os import logging import warnings @@ -8,7 +8,7 @@ import pytz import sqlalchemy -from pylons import config +from pylons import config as pylons_config import formencode import ckan.config.routing as routing @@ -22,7 +22,7 @@ import ckan.authz as authz import ckan.lib.jinja_extensions as jinja_extensions -from ckan.common import _, ungettext +from ckan.common import _, ungettext, config from ckan.exceptions import CkanConfigurationException log = logging.getLogger(__name__) @@ -72,8 +72,8 @@ def find_controller(self, controller): static_files=os.path.join(root, 'public'), templates=[]) - # Initialize config with the basic options - config.init_app(global_conf, app_conf, package='ckan', paths=paths) + # Initialize Pylons config with the basic options + pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths) # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings @@ -85,8 +85,9 @@ def find_controller(self, controller): warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) # load all CKAN plugins - p.load_all(config) + p.load_all() + app_globals.reset() # A mapping of config settings that can be overridden by env vars. # Note: Do not remove the following lines, they are used in the docs @@ -184,10 +185,11 @@ def update_config(): # The RoutesMiddleware needs its mapper updating if it exists if 'routes.middleware' in config: config['routes.middleware'].mapper = routes_map + # routes.named_routes is a CKAN thing config['routes.named_routes'] = routing.named_routes config['pylons.app_globals'] = app_globals.app_globals # initialise the globals - config['pylons.app_globals']._init() + app_globals.app_globals._init() helpers.load_plugin_helpers() config['pylons.h'] = helpers.helper_functions diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py index 33adb1be313..ac37b355775 100644 --- a/ckan/config/middleware/__init__.py +++ b/ckan/config/middleware/__init__.py @@ -7,6 +7,7 @@ from werkzeug.test import create_environ, run_wsgi_app from wsgi_party import WSGIParty +from ckan.config.environment import load_environment from ckan.config.middleware.flask_app import make_flask_stack from ckan.config.middleware.pylons_app import make_pylons_stack @@ -42,8 +43,10 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): middleware. ''' + load_environment(conf, app_conf) + pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) - flask_app = make_flask_stack(conf) + flask_app = make_flask_stack(conf, **app_conf) app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, 'flask_app': flask_app}) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index 8d37e26d1b8..7ff879d68ab 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -13,12 +13,16 @@ log = logging.getLogger(__name__) -def make_flask_stack(conf): +def make_flask_stack(conf, **app_conf): """ This has to pass the flask app through all the same middleware that Pylons used """ app = CKANFlask(__name__) + # Update Flask config with the CKAN values + app.config.update(conf) + app.config.update(app_conf) + @app.route('/hello', methods=['GET']) def hello_world(): return 'Hello World, this is served by Flask' diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py index 4d2af590c0c..faa173aca6c 100644 --- a/ckan/config/middleware/pylons_app.py +++ b/ckan/config/middleware/pylons_app.py @@ -19,8 +19,6 @@ from ckan.plugins import PluginImplementations from ckan.plugins.interfaces import IMiddleware import ckan.lib.uploader as uploader -from ckan.config.environment import load_environment -import ckan.lib.app_globals as app_globals from ckan.config.middleware import common_middleware import logging @@ -50,13 +48,8 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): defaults to main). """ - # Configure the Pylons environment - load_environment(conf, app_conf) - # The Pylons WSGI app app = PylonsApp() - # set pylons globals - app_globals.reset() for plugin in PluginImplementations(IMiddleware): app = plugin.make_middleware(app, config) @@ -71,7 +64,8 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) # app = QueueLogMiddleware(app) - if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): + if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', + True)): app = execute_on_completion(app, config, cleanup_pylons_response_string) @@ -137,11 +131,13 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): if asbool(static_files): # Serve static files - static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ + static_max_age = None if not asbool( + config.get('ckan.cache_enabled')) \ else int(config.get('ckan.static_max_age', 3600)) - static_app = StaticURLParser(config['pylons.paths']['static_files'], - cache_max_age=static_max_age) + static_app = StaticURLParser( + config['pylons.paths']['static_files'], + cache_max_age=static_max_age) static_parsers = [static_app, app] storage_directory = uploader.get_storage_path() @@ -159,7 +155,8 @@ def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): # Configurable extra static file paths extra_static_parsers = [] - for public_path in config.get('extra_public_paths', '').split(','): + for public_path in config.get( + 'extra_public_paths', '').split(','): if public_path.strip(): extra_static_parsers.append( StaticURLParser(public_path.strip(), diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index e62ec879537..f20b34ca08e 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -13,6 +13,7 @@ import ckan import ckan.model as model import ckan.logic as logic +from logic.schema import update_configuration_schema log = logging.getLogger(__name__) @@ -162,10 +163,11 @@ def get_config_value(key, default=''): # update the config config[key] = value + return value # update the config settings in auto update - schema = logic.schema.update_configuration_schema() + schema = update_configuration_schema() for key in schema.keys(): get_config_value(key) diff --git a/ckan/plugins/core.py b/ckan/plugins/core.py index f52b8ace507..07456fc8803 100644 --- a/ckan/plugins/core.py +++ b/ckan/plugins/core.py @@ -15,6 +15,9 @@ import interfaces +from ckan.common import config + + __all__ = [ 'PluginImplementations', 'implements', 'PluginNotFoundException', 'Plugin', 'SingletonPlugin', @@ -118,7 +121,7 @@ def plugins_update(): environment.update_config() -def load_all(config): +def load_all(): ''' Load all plugins listed in the 'ckan.plugins' config directive. ''' diff --git a/ckan/tests/legacy/test_plugins.py b/ckan/tests/legacy/test_plugins.py index 61343f12a57..15a0674735f 100644 --- a/ckan/tests/legacy/test_plugins.py +++ b/ckan/tests/legacy/test_plugins.py @@ -97,14 +97,14 @@ def test_plugins_load(self): config_plugins = config['ckan.plugins'] config['ckan.plugins'] = 'mapper_plugin routes_plugin' - plugins.load_all(config) + plugins.load_all() # synchronous_search automatically gets loaded current_plugins = set([plugins.get_plugin(p) for p in ['mapper_plugin', 'routes_plugin', 'synchronous_search'] + find_system_plugins()]) assert PluginGlobals.env().services == current_plugins # cleanup config['ckan.plugins'] = config_plugins - plugins.load_all(config) + plugins.load_all() def test_only_configured_plugins_loaded(self): with plugins.use_plugin('mapper_plugin') as p: @@ -119,7 +119,7 @@ def test_plugin_loading_order(self): """ config_plugins = config['ckan.plugins'] config['ckan.plugins'] = 'test_observer_plugin mapper_plugin mapper_plugin2' - plugins.load_all(config) + plugins.load_all() observerplugin = plugins.get_plugin('test_observer_plugin') @@ -133,7 +133,7 @@ def test_plugin_loading_order(self): assert observerplugin.after_load.calls[:3] == expected_order config['ckan.plugins'] = 'test_observer_plugin mapper_plugin2 mapper_plugin' - plugins.load_all(config) + plugins.load_all() expected_order = _make_calls(plugins.get_plugin('mapper_plugin2'), plugins.get_plugin('mapper_plugin')) @@ -144,7 +144,7 @@ def test_plugin_loading_order(self): assert observerplugin.after_load.calls[:3] == expected_order # cleanup config['ckan.plugins'] = config_plugins - plugins.load_all(config) + plugins.load_all() def test_mapper_plugin_fired_on_insert(self): with plugins.use_plugin('mapper_plugin') as mapper_plugin: From 4c377288a1a81b99a56f917d8b9e9a6ca8cdcd4f Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 12 Jul 2016 14:16:47 +0100 Subject: [PATCH 097/210] [#2842] Wrap Pylons requests in a Flask application context On this particular branch this is needed so the common CKAN config object can forward config options to the Flask app config object, but this will be required anyway as more Flask features need to be available during a Pylons request (see 62f55d2c96). --- ckan/config/middleware/__init__.py | 11 ++++++++++- ckan/config/middleware/flask_app.py | 5 ++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py index ac37b355775..05b3f006b63 100644 --- a/ckan/config/middleware/__init__.py +++ b/ckan/config/middleware/__init__.py @@ -132,4 +132,13 @@ def __call__(self, environ, start_response): log.debug('Serving request via {0} app'.format(app_name)) environ['ckan.app'] = app_name - return self.apps[app_name](environ, start_response) + if app_name == 'flask_app': + return self.apps[app_name](environ, start_response) + else: + # Although this request will be served by Pylons we still + # need an application context in order for the Flask URL + # builder to work and to be able to access the Flask config + flask_app = self.apps['flask_app']._wsgi_app + + with flask_app.app_context(): + return self.apps[app_name](environ, start_response) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index 7ff879d68ab..19091c4e29c 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -17,7 +17,7 @@ def make_flask_stack(conf, **app_conf): """ This has to pass the flask app through all the same middleware that Pylons used """ - app = CKANFlask(__name__) + app = flask_app = CKANFlask(__name__) # Update Flask config with the CKAN values app.config.update(conf) @@ -31,6 +31,9 @@ def hello_world(): def hello_world_post(): return 'Hello World, this was posted to Flask' + # Add a reference to the actual Flask app so it's easier to access + app._wsgi_app = flask_app + return app From ad46dce51cd570c814355c833efffb5d7e72cf59 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 13 Jul 2016 11:23:20 +0100 Subject: [PATCH 098/210] [#2842] New CKAN config object Rather than rely on the Pylons (or Flask) config object we define our own in ckan.common. This is a dict-like object (so fully backwards compatible) that also proxies any changes to the Flask and Pylons configuration objects (if they are available) This should be the only configuration object used in all code unless we really need to access the underlying Flask or Pylons objects for some reason. The `ckan.common.config` instance is initialized in the `load_environment` method with the values of the ini file or env vars. This is actually a proxy to a property from a Werkzeug Local object, meaning that `config` is thread-local safe, like its Flask and Pylons counterparts. See http://werkzeug.pocoo.org/docs/0.11/local/ I tried to separate all Pylons specific stuff (things like `pylons.paths`, `routes.map` etc) to Pylons own config object to keep the main config object clean but it proved too difficult, not only because we access these keys from different parts of the code (which can be solved) but also because when clearing the config (done on the tests) we lose keys that were added on `load_environment` and we would need to keep track of those. --- ckan/common.py | 81 +++++++++++ ckan/config/environment.py | 11 +- ckan/config/middleware/__init__.py | 3 +- ckan/config/middleware/flask_app.py | 12 +- ckan/config/middleware/pylons_app.py | 5 +- ckan/tests/test_common.py | 208 +++++++++++++++++++++++++++ 6 files changed, 313 insertions(+), 7 deletions(-) create mode 100644 ckan/tests/test_common.py diff --git a/ckan/common.py b/ckan/common.py index c711c1e48b5..cfb7409004c 100644 --- a/ckan/common.py +++ b/ckan/common.py @@ -8,6 +8,12 @@ # NOTE: This file is specificaly created for # from ckan.common import x, y, z to be allowed +from collections import MutableMapping + +import flask +import pylons + +from werkzeug.local import Local from pylons.i18n import _, ungettext from pylons import g, c, request, session, response @@ -17,3 +23,78 @@ from collections import OrderedDict # from python 2.7 except ImportError: from sqlalchemy.util import OrderedDict + + +class CKANConfig(MutableMapping): + u'''Main CKAN configuration object + + This is a dict-like object that also proxies any changes to the + Flask and Pylons configuration objects. + + The actual `config` instance in this module is initialized in the + `load_environment` method with the values of the ini file or env vars. + + ''' + + def __init__(self, *args, **kwargs): + self.store = dict() + self.update(dict(*args, **kwargs)) + + def __getitem__(self, key): + return self.store[key] + + def __iter__(self): + return iter(self.store) + + def __len__(self): + return len(self.store) + + def __repr__(self): + return self.store.__repr__() + + def copy(self): + return self.store.copy() + + def clear(self): + self.store.clear() + + try: + flask.current_app.config.clear() + except RuntimeError: + pass + try: + pylons.config.clear() + # Pylons set this default itself + pylons.config[u'lang'] = None + except TypeError: + pass + + def __setitem__(self, key, value): + self.store[key] = value + try: + flask.current_app.config[key] = value + except RuntimeError: + pass + try: + pylons.config[key] = value + except TypeError: + pass + + def __delitem__(self, key): + del self.store[key] + try: + del flask.current_app.config[key] + except RuntimeError: + pass + try: + del pylons.config[key] + except TypeError: + pass + +local = Local() + +# This a proxy to the bounded config object +local(u'config') + +# Thread-local safe objects +config = local.config = CKANConfig() diff --git a/ckan/config/environment.py b/ckan/config/environment.py index d668a11b87f..278e5fceb0b 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -72,9 +72,18 @@ def find_controller(self, controller): static_files=os.path.join(root, 'public'), templates=[]) - # Initialize Pylons config with the basic options + # Initialize main CKAN config object + config.update(global_conf) + config.update(app_conf) + + # Initialize Pylons own config object pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths) + # Update the main CKAN config object with the Pylons specific stuff, as it + # quite hard to keep them separated. This should be removed once Pylons + # support is dropped + config.update(pylons_config) + # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings msgs = ['^Unicode type received non-unicode bind param value', diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py index 05b3f006b63..f22e7818f14 100644 --- a/ckan/config/middleware/__init__.py +++ b/ckan/config/middleware/__init__.py @@ -45,7 +45,8 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf): load_environment(conf, app_conf) - pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) + pylons_app = make_pylons_stack(conf, full_stack, static_files, + **app_conf) flask_app = make_flask_stack(conf, **app_conf) app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py index 19091c4e29c..d24e62d90c5 100644 --- a/ckan/config/middleware/flask_app.py +++ b/ckan/config/middleware/flask_app.py @@ -8,6 +8,8 @@ from wsgi_party import WSGIParty, HighAndDry +from ckan.common import config + import logging log = logging.getLogger(__name__) @@ -19,9 +21,13 @@ def make_flask_stack(conf, **app_conf): app = flask_app = CKANFlask(__name__) - # Update Flask config with the CKAN values - app.config.update(conf) - app.config.update(app_conf) + # Update Flask config with the CKAN values. We use the common config + # object as values might have been modified on `load_environment` + if config: + app.config.update(config) + else: + app.config.update(conf) + app.config.update(app_conf) @app.route('/hello', methods=['GET']) def hello_world(): diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py index faa173aca6c..0d5f56f7023 100644 --- a/ckan/config/middleware/pylons_app.py +++ b/ckan/config/middleware/pylons_app.py @@ -2,7 +2,6 @@ import os -from pylons import config from pylons.wsgiapp import PylonsApp from beaker.middleware import CacheMiddleware, SessionMiddleware @@ -20,12 +19,14 @@ from ckan.plugins.interfaces import IMiddleware import ckan.lib.uploader as uploader from ckan.config.middleware import common_middleware +from ckan.common import config import logging log = logging.getLogger(__name__) -def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): +def make_pylons_stack(conf, full_stack=True, static_files=True, + **app_conf): """Create a Pylons WSGI application and return it ``conf`` diff --git a/ckan/tests/test_common.py b/ckan/tests/test_common.py new file mode 100644 index 00000000000..6ff25c416b9 --- /dev/null +++ b/ckan/tests/test_common.py @@ -0,0 +1,208 @@ +# encoding: utf-8 + +import flask +import pylons + +from nose.tools import eq_, assert_not_equal as neq_ + +from ckan.tests import helpers +from ckan.common import CKANConfig, config as ckan_config + + +class TestConfigObject(object): + + def test_del_works(self): + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + + del my_conf[u'test_key_1'] + + assert u'test_key_1' not in my_conf + + def test_get_item_works(self): + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + + eq_(my_conf.get(u'test_key_1'), u'Test value 1') + + def test_repr_works(self): + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + + eq_(repr(my_conf), u"{u'test_key_1': u'Test value 1'}") + + def test_len_works(self): + + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + my_conf[u'test_key_2'] = u'Test value 2' + + eq_(len(my_conf), 2) + + def test_keys_works(self): + + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + my_conf[u'test_key_2'] = u'Test value 2' + + eq_(sorted(my_conf.keys()), [u'test_key_1', u'test_key_2']) + + def test_clear_works(self): + + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + my_conf[u'test_key_2'] = u'Test value 2' + + eq_(len(my_conf.keys()), 2) + + my_conf.clear() + + eq_(len(my_conf.keys()), 0) + + def test_for_in_works(self): + + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + my_conf[u'test_key_2'] = u'Test value 2' + + cnt = 0 + for key in my_conf: + cnt += 1 + assert key.startswith(u'test_key_') + + eq_(cnt, 2) + + def test_iteritems_works(self): + + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + my_conf[u'test_key_2'] = u'Test value 2' + + cnt = 0 + for key, value in my_conf.iteritems(): + cnt += 1 + assert key.startswith(u'test_key_') + assert value.startswith(u'Test value') + + eq_(cnt, 2) + + def test_not_true_if_empty(self): + + my_conf = CKANConfig() + + assert not my_conf + + def test_true_if_not_empty(self): + + my_conf = CKANConfig() + + my_conf[u'test_key_1'] = u'Test value 1' + + assert my_conf + + +class TestCommonConfig(object): + + def setup(self): + self._original_config = ckan_config.copy() + + def teardown(self): + ckan_config.clear() + ckan_config.update(self._original_config) + + @classmethod + def teardown_class(cls): + helpers.reset_db() + + def test_setting_a_key_sets_it_on_pylons_config(self): + + ckan_config[u'ckan.site_title'] = u'Example title' + eq_(pylons.config[u'ckan.site_title'], u'Example title') + + def test_setting_a_key_sets_it_on_flask_config_if_app_context(self): + + app = helpers._get_test_app() + with app.flask_app.app_context(): + + ckan_config[u'ckan.site_title'] = u'Example title' + eq_(flask.current_app.config[u'ckan.site_title'], u'Example title') + + def test_setting_a_key_does_not_set_it_on_flask_config_if_outside_app_context(self): + + ckan_config[u'ckan.site_title'] = u'Example title' + + app = helpers._get_test_app() + with app.flask_app.app_context(): + + neq_(flask.current_app.config[u'ckan.site_title'], u'Example title') + + def test_deleting_a_key_deletes_it_on_pylons_config(self): + + ckan_config[u'ckan.site_title'] = u'Example title' + del ckan_config[u'ckan.site_title'] + + assert u'ckan.site_title' not in ckan_config + + def test_deleting_a_key_delets_it_on_flask_config(self): + + app = helpers._get_test_app() + with app.flask_app.app_context(): + + ckan_config[u'ckan.site_title'] = u'Example title' + del ckan_config[u'ckan.site_title'] + + assert u'ckan.site_title' not in flask.current_app.config + + def test_update_works_on_pylons_config(self): + + ckan_config[u'ckan.site_title'] = u'Example title' + + ckan_config.update({ + u'ckan.site_title': u'Example title 2', + u'ckan.new_key': u'test'}) + + eq_(pylons.config[u'ckan.site_title'], u'Example title 2') + eq_(pylons.config[u'ckan.new_key'], u'test') + + def test_update_works_on_flask_config(self): + + app = helpers._get_test_app() + with app.flask_app.app_context(): + + ckan_config[u'ckan.site_title'] = u'Example title' + + ckan_config.update({ + u'ckan.site_title': u'Example title 2', + u'ckan.new_key': u'test'}) + + eq_(flask.current_app.config[u'ckan.site_title'], u'Example title 2') + eq_(flask.current_app.config[u'ckan.new_key'], u'test') + + def test_config_option_update_action_works_on_pylons(self): + params = { + u'ckan.site_title': u'Example title action', + } + + helpers.call_action(u'config_option_update', {}, **params) + + eq_(pylons.config[u'ckan.site_title'], u'Example title action') + + def test_config_option_update_action_works_on_flask(self): + app = helpers._get_test_app() + with app.flask_app.app_context(): + + params = { + u'ckan.site_title': u'Example title action', + } + + helpers.call_action(u'config_option_update', {}, **params) + + eq_(pylons.config[u'ckan.site_title'], u'Example title action') From a77ff9c021c80eee949d2ed11cbafd743ee5c04a Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 13 Jul 2016 11:49:19 +0100 Subject: [PATCH 099/210] [#2842] Update all core imports to ckan.common.config --- ckan/authz.py | 2 +- ckan/ckan_nose_plugin.py | 2 +- ckan/controllers/admin.py | 2 +- ckan/controllers/feed.py | 2 +- ckan/controllers/home.py | 2 +- ckan/controllers/package.py | 2 +- ckan/controllers/tag.py | 2 +- ckan/controllers/user.py | 2 +- .../lib/activity_streams_session_extension.py | 2 +- ckan/lib/app_globals.py | 2 +- ckan/lib/auth_tkt.py | 2 +- ckan/lib/base.py | 6 ++-- ckan/lib/captcha.py | 2 +- ckan/lib/celery_app.py | 4 +-- ckan/lib/cli.py | 8 ++--- ckan/lib/datapreview.py | 2 +- ckan/lib/dictization/__init__.py | 2 +- ckan/lib/dictization/model_dictize.py | 2 +- ckan/lib/email_notifications.py | 9 ++--- ckan/lib/hash.py | 2 +- ckan/lib/helpers.py | 2 +- ckan/lib/i18n.py | 5 +-- ckan/lib/mailer.py | 2 +- ckan/lib/navl/dictization_functions.py | 2 +- ckan/lib/search/__init__.py | 2 +- ckan/lib/search/index.py | 2 +- ckan/lib/search/query.py | 2 +- ckan/lib/uploader.py | 3 +- ckan/logic/action/create.py | 3 +- ckan/logic/action/get.py | 2 +- ckan/logic/action/update.py | 4 +-- .../versions/081_set_datastore_active.py | 2 +- ckan/model/license.py | 2 +- ckan/model/meta.py | 2 +- ckan/model/package.py | 2 +- ckan/model/resource.py | 2 +- ckan/plugins/interfaces.py | 4 +-- ckan/plugins/toolkit.py | 1 + ckan/tests/config/test_environment.py | 2 +- ckan/tests/controllers/test_admin.py | 2 +- ckan/tests/helpers.py | 23 ++++++++++--- ckan/tests/legacy/__init__.py | 2 +- ckan/tests/legacy/functional/api/base.py | 2 +- .../functional/api/model/test_package.py | 4 +-- .../legacy/functional/api/test_activity.py | 2 +- .../api/test_email_notifications.py | 3 +- ckan/tests/legacy/functional/api/test_user.py | 2 +- ckan/tests/legacy/functional/test_activity.py | 2 +- ckan/tests/legacy/functional/test_package.py | 4 +-- ckan/tests/legacy/functional/test_user.py | 2 +- ckan/tests/legacy/lib/test_helpers.py | 2 +- ckan/tests/legacy/lib/test_i18n.py | 3 +- .../legacy/lib/test_solr_search_index.py | 2 +- ckan/tests/legacy/logic/test_action.py | 2 +- ckan/tests/legacy/logic/test_auth.py | 2 +- .../legacy/misc/test_mock_mail_server.py | 2 +- ckan/tests/legacy/misc/test_sync.py | 2 +- ckan/tests/legacy/mock_mail_server.py | 2 +- ckan/tests/legacy/test_plugins.py | 2 +- ckan/tests/lib/search/test_index.py | 2 +- ckan/tests/lib/test_datapreview.py | 2 +- ckan/tests/lib/test_mailer.py | 2 +- ckan/tests/logic/action/test_create.py | 2 +- ckan/tests/logic/action/test_patch.py | 2 +- ckan/tests/logic/action/test_update.py | 2 +- ckan/tests/model/test_license.py | 2 +- ckanext/datapusher/logic/action.py | 8 ++--- ckanext/datapusher/tests/test.py | 10 +++--- ckanext/datapusher/tests/test_interfaces.py | 2 +- ckanext/datastore/db.py | 16 ++++----- ckanext/datastore/logic/action.py | 21 ++++++------ ckanext/datastore/plugin.py | 4 +-- ckanext/datastore/tests/test_create.py | 7 ++-- ckanext/datastore/tests/test_delete.py | 4 +-- ckanext/datastore/tests/test_disable.py | 2 +- ckanext/datastore/tests/test_dump.py | 2 +- ckanext/datastore/tests/test_helpers.py | 6 ++-- ckanext/datastore/tests/test_search.py | 8 ++--- ckanext/datastore/tests/test_unit.py | 5 ++- ckanext/datastore/tests/test_upsert.py | 8 ++--- .../plugin_v5_custom_config_setting.py | 2 +- .../plugin_v6_parent_auth_functions.py | 2 +- .../tests/test_example_iauthfunctions.py | 6 ++-- .../tests/test_iconfigurer_update_config.py | 2 +- .../tests/test_example_idatasetform.py | 2 +- .../tests/test_example_iresourcecontroller.py | 2 +- ckanext/example_iuploader/test/test_plugin.py | 2 +- .../tests/test_ivalidators.py | 2 +- ckanext/multilingual/plugin.py | 33 +++++++++---------- ckanext/reclineview/tests/test_view.py | 2 +- ckanext/resourceproxy/controller.py | 2 +- ckanext/resourceproxy/plugin.py | 2 +- ckanext/resourceproxy/tests/test_proxy.py | 2 +- ckanext/stats/stats.py | 2 +- ckanext/stats/tests/__init__.py | 2 +- ckanext/stats/tests/test_stats_plugin.py | 2 +- ckanext/textview/tests/test_view.py | 2 +- doc/extensions/custom-config-settings.rst | 2 +- 98 files changed, 184 insertions(+), 176 deletions(-) diff --git a/ckan/authz.py b/ckan/authz.py index 4f169e0480a..d77f91b887d 100644 --- a/ckan/authz.py +++ b/ckan/authz.py @@ -4,7 +4,7 @@ import re from logging import getLogger -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool import ckan.plugins as p diff --git a/ckan/ckan_nose_plugin.py b/ckan/ckan_nose_plugin.py index ed5bb3015be..c0f922f479c 100644 --- a/ckan/ckan_nose_plugin.py +++ b/ckan/ckan_nose_plugin.py @@ -8,7 +8,7 @@ import re import pkg_resources from paste.deploy import loadapp -from pylons import config +from ckan.common import config import unittest import time diff --git a/ckan/controllers/admin.py b/ckan/controllers/admin.py index d6768d964c9..18eccd16163 100644 --- a/ckan/controllers/admin.py +++ b/ckan/controllers/admin.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from pylons import config +from ckan.common import config import ckan.lib.base as base import ckan.lib.helpers as h diff --git a/ckan/controllers/feed.py b/ckan/controllers/feed.py index 3b41af9d80d..4686f1cdba3 100644 --- a/ckan/controllers/feed.py +++ b/ckan/controllers/feed.py @@ -25,7 +25,7 @@ import urlparse import webhelpers.feedgenerator -from pylons import config +from ckan.common import config import ckan.model as model import ckan.lib.base as base diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index 4a8cdbb26d4..e1bdfadce9f 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from pylons import config, cache +from pylons import cache import sqlalchemy.exc import ckan.logic as logic diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 5b41aa95e81..27e1b83a3b4 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -6,7 +6,7 @@ import mimetypes import cgi -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool import paste.fileapp diff --git a/ckan/controllers/tag.py b/ckan/controllers/tag.py index 67b761af444..d3b62fcdec3 100644 --- a/ckan/controllers/tag.py +++ b/ckan/controllers/tag.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool import ckan.logic as logic diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index c7637fa122d..6acb42ce650 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -3,7 +3,7 @@ import logging from urllib import quote -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool import ckan.lib.base as base diff --git a/ckan/lib/activity_streams_session_extension.py b/ckan/lib/activity_streams_session_extension.py index 6490658f0d2..1a940d62cac 100644 --- a/ckan/lib/activity_streams_session_extension.py +++ b/ckan/lib/activity_streams_session_extension.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from pylons import config +from ckan.common import config from sqlalchemy.orm.session import SessionExtension from paste.deploy.converters import asbool import logging diff --git a/ckan/lib/app_globals.py b/ckan/lib/app_globals.py index f20b34ca08e..31054ae2578 100644 --- a/ckan/lib/app_globals.py +++ b/ckan/lib/app_globals.py @@ -8,7 +8,7 @@ import re from paste.deploy.converters import asbool -from pylons import config +from ckan.common import config import ckan import ckan.model as model diff --git a/ckan/lib/auth_tkt.py b/ckan/lib/auth_tkt.py index fee688c34c3..9f82e10dbd8 100644 --- a/ckan/lib/auth_tkt.py +++ b/ckan/lib/auth_tkt.py @@ -3,7 +3,7 @@ import math import os -from pylons import config +from ckan.common import config from repoze.who.plugins import auth_tkt as repoze_auth_tkt _bool = repoze_auth_tkt._bool diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 86b10c1399c..0894455817a 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -8,7 +8,7 @@ import time from paste.deploy.converters import asbool -from pylons import cache, config, session +from pylons import cache, session from pylons.controllers import WSGIController from pylons.controllers.util import abort as _abort from pylons.controllers.util import redirect_to, redirect @@ -30,7 +30,7 @@ # These imports are for legacy usages and will be removed soon these should # be imported directly from ckan.common for internal ckan code and via the # plugins.toolkit for extensions. -from ckan.common import json, _, ungettext, c, g, request, response +from ckan.common import json, _, ungettext, c, g, request, response, config log = logging.getLogger(__name__) @@ -339,6 +339,7 @@ def _set_cors(self): True, or the request Origin is in the origin_whitelist. ''' cors_origin_allowed = None + if asbool(config.get('ckan.cors.origin_allow_all')): cors_origin_allowed = "*" elif config.get('ckan.cors.origin_whitelist') and \ @@ -346,7 +347,6 @@ def _set_cors(self): in config['ckan.cors.origin_whitelist'].split(): # set var to the origin to allow it. cors_origin_allowed = request.headers.get('Origin') - if cors_origin_allowed is not None: response.headers['Access-Control-Allow-Origin'] = \ cors_origin_allowed diff --git a/ckan/lib/captcha.py b/ckan/lib/captcha.py index b6749a3d300..bee39f68406 100644 --- a/ckan/lib/captcha.py +++ b/ckan/lib/captcha.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from pylons import config +from ckan.common import config import urllib import urllib2 diff --git a/ckan/lib/celery_app.py b/ckan/lib/celery_app.py index ee648724cdd..9f66e85f33b 100644 --- a/ckan/lib/celery_app.py +++ b/ckan/lib/celery_app.py @@ -4,7 +4,7 @@ import os import logging -from pylons import config as pylons_config +from ckan.common import config as ckan_config from pkg_resources import iter_entry_points, VersionConflict log = logging.getLogger(__name__) @@ -25,7 +25,7 @@ config.read(config_file) -sqlalchemy_url = pylons_config.get('sqlalchemy.url') +sqlalchemy_url = ckan_config.get('sqlalchemy.url') if not sqlalchemy_url: sqlalchemy_url = config.get('app:main', 'sqlalchemy.url') diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 86537863c3b..f2dd886dccf 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -20,7 +20,7 @@ import sqlalchemy as sa import urlparse import routes -from pylons import config +from ckan.common import config import paste.script from paste.registry import Registry @@ -37,7 +37,7 @@ def parse_db_config(config_key='sqlalchemy.url'): 'postgres://tester:pass@localhost/ckantest3' ''' - from pylons import config + from ckan.common import config url = config[config_key] regex = [ '^\s*(?P\w*)', @@ -569,7 +569,7 @@ def export_datasets(self, out_folder): ''' import urlparse import urllib2 - import pylons.config as config + from ckan.common import config import ckan.model as model import ckan.logic as logic import ckan.lib.helpers as h @@ -1759,7 +1759,7 @@ class TranslationsCommand(CkanCommand): def command(self): self._load_config() - from pylons import config + from ckan.common import config self.ckan_path = os.path.join(os.path.dirname(__file__), '..') i18n_path = os.path.join(self.ckan_path, 'i18n') self.i18n_path = config.get('ckan.i18n_directory', i18n_path) diff --git a/ckan/lib/datapreview.py b/ckan/lib/datapreview.py index e17168e14fa..cb995de2437 100644 --- a/ckan/lib/datapreview.py +++ b/ckan/lib/datapreview.py @@ -8,7 +8,7 @@ import urlparse import logging -import pylons.config as config +from ckan.common import config import ckan.plugins as p from ckan import logic diff --git a/ckan/lib/dictization/__init__.py b/ckan/lib/dictization/__init__.py index 96c4c281a9f..2df79363231 100644 --- a/ckan/lib/dictization/__init__.py +++ b/ckan/lib/dictization/__init__.py @@ -3,7 +3,7 @@ import datetime from sqlalchemy.orm import class_mapper import sqlalchemy -from pylons import config +from ckan.common import config try: RowProxy = sqlalchemy.engine.result.RowProxy diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index bf38416f493..0bbfd46d940 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -14,7 +14,7 @@ import datetime import urlparse -from pylons import config +from ckan.common import config from sqlalchemy.sql import select import ckan.logic as logic diff --git a/ckan/lib/email_notifications.py b/ckan/lib/email_notifications.py index 20a6965f5f9..3d05228d632 100644 --- a/ckan/lib/email_notifications.py +++ b/ckan/lib/email_notifications.py @@ -9,13 +9,11 @@ import datetime import re -import pylons - import ckan.model as model import ckan.logic as logic import ckan.lib.base as base -from ckan.common import ungettext +from ckan.common import ungettext, config def string_to_timedelta(s): @@ -103,7 +101,7 @@ def _notifications_for_activities(activities, user_dict): "{n} new activity from {site_title}", "{n} new activities from {site_title}", len(activities)).format( - site_title=pylons.config.get('ckan.site_title'), + site_title=config.get('ckan.site_title'), n=len(activities)) body = base.render( 'activity_streams/activity_stream_email_notifications.text', @@ -190,7 +188,7 @@ def get_and_send_notifications_for_user(user): # Parse the email_notifications_since config setting, email notifications # from longer ago than this time will not be sent. - email_notifications_since = pylons.config.get( + email_notifications_since = config.get( 'ckan.email_notifications_since', '2 days') email_notifications_since = string_to_timedelta( email_notifications_since) @@ -208,7 +206,6 @@ def get_and_send_notifications_for_user(user): activity_stream_last_viewed) notifications = get_notifications(user, since) - # TODO: Handle failures from send_email_notification. for notification in notifications: send_notification(user, notification) diff --git a/ckan/lib/hash.py b/ckan/lib/hash.py index 5f423ee650a..87a15ab7348 100644 --- a/ckan/lib/hash.py +++ b/ckan/lib/hash.py @@ -3,7 +3,7 @@ import hmac import hashlib -from pylons import config, request +from ckan.common import config, request secret = None diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 5ae9929416a..287d758aac0 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -26,7 +26,7 @@ from markdown import markdown from bleach import clean as clean_html from pylons import url as _pylons_default_url -from pylons import config +from ckan.common import config from routes import redirect_to as _redirect_to from routes import url_for as _routes_default_url_for import i18n diff --git a/ckan/lib/i18n.py b/ckan/lib/i18n.py index 9a04a610c3d..5fb13e1cec5 100644 --- a/ckan/lib/i18n.py +++ b/ckan/lib/i18n.py @@ -8,10 +8,11 @@ UnknownLocaleError) from babel.support import Translations from paste.deploy.converters import aslist -from pylons import config from pylons import i18n import pylons + +from ckan.common import config import ckan.i18n from ckan.plugins import PluginImplementations from ckan.plugins.interfaces import ITranslation @@ -173,7 +174,7 @@ def _set_lang(lang): if config.get('ckan.i18n_directory'): fake_config = {'pylons.paths': {'root': config['ckan.i18n_directory']}, 'pylons.package': config['pylons.package']} - i18n.set_lang(lang, pylons_config=fake_config, class_=Translations) + i18n.set_lang(lang, config=fake_config, class_=Translations) else: i18n.set_lang(lang, class_=Translations) diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index a93242fd15b..1efa7a3bd3a 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -9,7 +9,7 @@ from email import Utils from urlparse import urljoin -from pylons import config +from ckan.common import config import paste.deploy.converters import ckan diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py index d106008ee09..5461a74eb92 100644 --- a/ckan/lib/navl/dictization_functions.py +++ b/ckan/lib/navl/dictization_functions.py @@ -4,7 +4,7 @@ import formencode as fe import inspect import json -from pylons import config +from ckan.common import config from ckan.common import _ diff --git a/ckan/lib/search/__init__.py b/ckan/lib/search/__init__.py index 7807f430bd5..65f064093a2 100644 --- a/ckan/lib/search/__init__.py +++ b/ckan/lib/search/__init__.py @@ -7,7 +7,7 @@ import xml.dom.minidom import urllib2 -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool import ckan.model as model diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py index 5bdf82c0796..878a5fa3a8c 100644 --- a/ckan/lib/search/index.py +++ b/ckan/lib/search/index.py @@ -11,7 +11,7 @@ import re import pysolr -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool from common import SearchIndexError, make_connection diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 15607e0baa7..c136b294ca4 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -3,7 +3,7 @@ import re import logging -from pylons import config +from ckan.common import config import pysolr from paste.deploy.converters import asbool from paste.util.multidict import MultiDict diff --git a/ckan/lib/uploader.py b/ckan/lib/uploader.py index 9164cdbb1f2..d65d15a3888 100644 --- a/ckan/lib/uploader.py +++ b/ckan/lib/uploader.py @@ -2,15 +2,14 @@ import os import cgi -import pylons import datetime import logging import ckan.lib.munge as munge import ckan.logic as logic import ckan.plugins as plugins +from ckan.common import config -config = pylons.config log = logging.getLogger(__name__) _storage_path = None diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index b85ef81f27b..799f1064430 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -7,7 +7,6 @@ import re from socket import error as socket_error -from pylons import config import paste.deploy.converters from sqlalchemy import func @@ -25,7 +24,7 @@ import ckan.lib.mailer as mailer import ckan.lib.datapreview -from ckan.common import _ +from ckan.common import _, config # FIXME this looks nasty and should be shared better from ckan.logic.action.update import _update_package_relationship diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 20d765e5654..8ba39c1d43a 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -8,7 +8,7 @@ import datetime import socket -from pylons import config +from ckan.common import config import sqlalchemy from paste.deploy.converters import asbool diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 798d51103b6..ddf17979901 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -7,7 +7,7 @@ import time import json -from pylons import config +from ckan.common import config import paste.deploy.converters as converters import ckan.plugins as plugins @@ -1248,7 +1248,7 @@ def config_option_update(context, data_dict): # Save value in database model.set_system_info(key, value) - # Update the pylons `config` object + # Update CKAN's `config` object config[key] = value # Only add it to the app_globals (`g`) object if explicitly defined diff --git a/ckan/migration/versions/081_set_datastore_active.py b/ckan/migration/versions/081_set_datastore_active.py index 9038450d887..a8219800778 100644 --- a/ckan/migration/versions/081_set_datastore_active.py +++ b/ckan/migration/versions/081_set_datastore_active.py @@ -4,7 +4,7 @@ from sqlalchemy import create_engine from sqlalchemy.sql import text from sqlalchemy.exc import SQLAlchemyError -from pylons import config +from ckan.common import config def upgrade(migrate_engine): diff --git a/ckan/model/license.py b/ckan/model/license.py index 6395f79d4ab..0f2f2b9279e 100644 --- a/ckan/model/license.py +++ b/ckan/model/license.py @@ -4,7 +4,7 @@ import urllib2 import re -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool from ckan.common import _, json diff --git a/ckan/model/meta.py b/ckan/model/meta.py index 7de9773bee5..e4a7429d99f 100644 --- a/ckan/model/meta.py +++ b/ckan/model/meta.py @@ -3,7 +3,7 @@ import datetime from paste.deploy.converters import asbool -from pylons import config +from ckan.common import config """SQLAlchemy Metadata and Session object""" from sqlalchemy import MetaData, and_ import sqlalchemy.orm as orm diff --git a/ckan/model/package.py b/ckan/model/package.py index 6526dc602d8..9ef68277330 100644 --- a/ckan/model/package.py +++ b/ckan/model/package.py @@ -8,7 +8,7 @@ from sqlalchemy.sql import select, and_, union, or_ from sqlalchemy import orm from sqlalchemy import types, Column, Table -from pylons import config +from ckan.common import config import vdm.sqlalchemy import meta diff --git a/ckan/model/resource.py b/ckan/model/resource.py index c7093576953..03a8b512015 100644 --- a/ckan/model/resource.py +++ b/ckan/model/resource.py @@ -5,7 +5,7 @@ from sqlalchemy.util import OrderedDict from sqlalchemy.ext.orderinglist import ordering_list from sqlalchemy import orm -from pylons import config +from ckan.common import config import vdm.sqlalchemy import vdm.sqlalchemy.stateful from sqlalchemy import types, func, Column, Table, ForeignKey, and_ diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index b2907c8076a..2651c394a87 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -747,7 +747,7 @@ def configure(self, config): class IConfigurer(Interface): """ - Configure CKAN (pylons) environment via the ``pylons.config`` object + Configure CKAN environment via the ``config`` object """ def update_config(self, config): @@ -755,7 +755,7 @@ def update_config(self, config): Called by load_environment at earliest point when config is available to plugins. The config should be updated in place. - :param config: ``pylons.config`` object + :param config: ``config`` object """ def update_config_schema(self, schema): diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 06a63722a24..427fc489fbb 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -356,6 +356,7 @@ def _add_ckan_admin_tabs(cls, config, route_name, tab_label, # update the config with the updated admin_tabs dict config.update({config_var: admin_tabs_dict}) +# import ipdb; ipdb.set_trace() @classmethod def _version_str_2_list(cls, v_str): ''' convert a version string into a list of ints diff --git a/ckan/tests/config/test_environment.py b/ckan/tests/config/test_environment.py index f9c5533a89e..4212fb5dfe0 100644 --- a/ckan/tests/config/test_environment.py +++ b/ckan/tests/config/test_environment.py @@ -3,7 +3,7 @@ import os from nose import tools as nosetools -from pylons import config +from ckan.common import config import ckan.tests.helpers as h import ckan.plugins as p diff --git a/ckan/tests/controllers/test_admin.py b/ckan/tests/controllers/test_admin.py index 3e1d623011f..8915c489d92 100644 --- a/ckan/tests/controllers/test_admin.py +++ b/ckan/tests/controllers/test_admin.py @@ -4,7 +4,7 @@ from bs4 import BeautifulSoup from routes import url_for -from pylons import config +from ckan.common import config import ckan.model as model import ckan.tests.helpers as helpers diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 7f8a0ff77c5..bfda83a1a1e 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -20,11 +20,11 @@ ''' import webtest -from pylons import config import nose.tools from nose.tools import assert_in, assert_not_in import mock +from ckan.common import config import ckan.lib.search as search import ckan.config.middleware import ckan.model as model @@ -132,6 +132,21 @@ def call_auth(auth_name, context, **kwargs): return logic.check_access(auth_name, context, data_dict=kwargs) +class CKANTestApp(webtest.TestApp): + '''A wrapper around webtest.TestApp + + It adds some convenience methods for CKAN + ''' + + _flask_app = None + + @property + def flask_app(self): + if not self._flask_app: + self._flask_app = self.app.apps['flask_app']._wsgi_app + return self._flask_app + + def _get_test_app(): '''Return a webtest.TestApp for CKAN, with legacy templates disabled. @@ -141,7 +156,7 @@ def _get_test_app(): ''' config['ckan.legacy_templates'] = False app = ckan.config.middleware.make_app(config['global_conf'], **config) - app = webtest.TestApp(app) + app = CKANTestApp(app) return app @@ -284,7 +299,7 @@ def webtest_maybe_follow(response, **kw): def change_config(key, value): - '''Decorator to temporarily changes Pylons' config to a new value + '''Decorator to temporarily change CKAN's config to a new value This allows you to easily create tests that need specific config values to be set, making sure it'll be reverted to what it was originally, after your @@ -294,7 +309,7 @@ def change_config(key, value): @helpers.change_config('ckan.site_title', 'My Test CKAN') def test_ckan_site_title(self): - assert pylons.config['ckan.site_title'] == 'My Test CKAN' + assert config['ckan.site_title'] == 'My Test CKAN' :param key: the config key to be changed, e.g. ``'ckan.site_title'`` :type key: string diff --git a/ckan/tests/legacy/__init__.py b/ckan/tests/legacy/__init__.py index fcbf89c7134..3a056855f95 100644 --- a/ckan/tests/legacy/__init__.py +++ b/ckan/tests/legacy/__init__.py @@ -18,7 +18,7 @@ from nose.plugins.skip import SkipTest import time -from pylons import config +from ckan.common import config from pylons.test import pylonsapp from paste.script.appinstall import SetupCommand diff --git a/ckan/tests/legacy/functional/api/base.py b/ckan/tests/legacy/functional/api/base.py index 9948d0ff3d4..fefedae0f65 100644 --- a/ckan/tests/legacy/functional/api/base.py +++ b/ckan/tests/legacy/functional/api/base.py @@ -8,7 +8,7 @@ import urllib -from pylons import config +from ckan.common import config import webhelpers.util from nose.tools import assert_equal from paste.fixture import TestRequest diff --git a/ckan/tests/legacy/functional/api/model/test_package.py b/ckan/tests/legacy/functional/api/model/test_package.py index ddf83a328cd..22472d7dac3 100644 --- a/ckan/tests/legacy/functional/api/model/test_package.py +++ b/ckan/tests/legacy/functional/api/model/test_package.py @@ -253,7 +253,7 @@ def test_register_post_indexerror(self): """ Test that we can't add a package if Solr is down. """ - bad_solr_url = 'http://127.0.0.1/badsolrurl' + bad_solr_url = 'http://example.com/badsolrurl' original_settings = SolrSettings.get()[0] try: SolrSettings.init(bad_solr_url) @@ -643,7 +643,7 @@ def test_entity_update_indexerror(self): """ Test that we can't update a package if Solr is down. """ - bad_solr_url = 'http://127.0.0.1/badsolrurl' + bad_solr_url = 'http://example.com/badsolrurl' original_settings = SolrSettings.get()[0] try: SolrSettings.init(bad_solr_url) diff --git a/ckan/tests/legacy/functional/api/test_activity.py b/ckan/tests/legacy/functional/api/test_activity.py index 33b614713a6..1350b7acaba 100644 --- a/ckan/tests/legacy/functional/api/test_activity.py +++ b/ckan/tests/legacy/functional/api/test_activity.py @@ -15,7 +15,7 @@ logger = logging.getLogger(__name__) import pylons.test -from pylons import config +from ckan.common import config from paste.deploy.converters import asbool import paste.fixture from nose import SkipTest diff --git a/ckan/tests/legacy/functional/api/test_email_notifications.py b/ckan/tests/legacy/functional/api/test_email_notifications.py index fe562756570..35bea72dbcb 100644 --- a/ckan/tests/legacy/functional/api/test_email_notifications.py +++ b/ckan/tests/legacy/functional/api/test_email_notifications.py @@ -14,7 +14,8 @@ import paste.deploy import pylons.test -from pylons import config +from ckan.common import config + class TestEmailNotifications(mock_mail_server.SmtpServerHarness, pylons_controller.PylonsTestCase): diff --git a/ckan/tests/legacy/functional/api/test_user.py b/ckan/tests/legacy/functional/api/test_user.py index 846ae8b9efe..e470ce2d8bd 100644 --- a/ckan/tests/legacy/functional/api/test_user.py +++ b/ckan/tests/legacy/functional/api/test_user.py @@ -1,7 +1,7 @@ # encoding: utf-8 import paste -from pylons import config +from ckan.common import config from nose.tools import assert_equal import ckan.logic as logic diff --git a/ckan/tests/legacy/functional/test_activity.py b/ckan/tests/legacy/functional/test_activity.py index dccdf303bad..f40f1397d78 100644 --- a/ckan/tests/legacy/functional/test_activity.py +++ b/ckan/tests/legacy/functional/test_activity.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from pylons import config +from ckan.common import config from pylons.test import pylonsapp from paste.deploy.converters import asbool import paste.fixture diff --git a/ckan/tests/legacy/functional/test_package.py b/ckan/tests/legacy/functional/test_package.py index ee342985768..b09a0f55ef9 100644 --- a/ckan/tests/legacy/functional/test_package.py +++ b/ckan/tests/legacy/functional/test_package.py @@ -2,7 +2,7 @@ import datetime -from pylons import config, c +from ckan.common import config, c from difflib import unified_diff from nose.tools import assert_equal @@ -612,7 +612,7 @@ def test_after_create_plugin_hook(self): plugins.unload('test_package_controller_plugin') def test_new_indexerror(self): - bad_solr_url = 'http://127.0.0.1/badsolrurl' + bad_solr_url = 'http://example.com/badsolrurl' solr_url = SolrSettings.get()[0] try: SolrSettings.init(bad_solr_url) diff --git a/ckan/tests/legacy/functional/test_user.py b/ckan/tests/legacy/functional/test_user.py index 2aa5e6f7abe..f61b8be35c7 100644 --- a/ckan/tests/legacy/functional/test_user.py +++ b/ckan/tests/legacy/functional/test_user.py @@ -2,7 +2,7 @@ from routes import url_for from nose.tools import assert_equal -from pylons import config +from ckan.common import config import hashlib from ckan.tests.legacy import CreateTestData diff --git a/ckan/tests/legacy/lib/test_helpers.py b/ckan/tests/legacy/lib/test_helpers.py index fe5d0ad7f1b..b0bfb9ba20b 100644 --- a/ckan/tests/legacy/lib/test_helpers.py +++ b/ckan/tests/legacy/lib/test_helpers.py @@ -3,7 +3,7 @@ import datetime from nose.tools import assert_equal, assert_raises -from pylons import config +from ckan.common import config from ckan.tests.legacy import * import ckan.lib.helpers as h diff --git a/ckan/tests/legacy/lib/test_i18n.py b/ckan/tests/legacy/lib/test_i18n.py index 3851fb8df2e..91e78f87258 100644 --- a/ckan/tests/legacy/lib/test_i18n.py +++ b/ckan/tests/legacy/lib/test_i18n.py @@ -1,10 +1,11 @@ # encoding: utf-8 from nose.tools import assert_equal -from pylons import config, session +from pylons import session import pylons import ckan.lib.i18n +from ckan.common import config from ckan.tests.legacy.pylons_controller import PylonsTestCase diff --git a/ckan/tests/legacy/lib/test_solr_search_index.py b/ckan/tests/legacy/lib/test_solr_search_index.py index e1bbce3a228..55a6d5977f7 100644 --- a/ckan/tests/legacy/lib/test_solr_search_index.py +++ b/ckan/tests/legacy/lib/test_solr_search_index.py @@ -1,7 +1,7 @@ # encoding: utf-8 import pysolr -from pylons import config +from ckan.common import config from ckan import model import ckan.lib.search as search from ckan.tests.legacy import TestController, CreateTestData, setup_test_search_index, is_search_supported diff --git a/ckan/tests/legacy/logic/test_action.py b/ckan/tests/legacy/logic/test_action.py index 558b7c20407..10f287beefb 100644 --- a/ckan/tests/legacy/logic/test_action.py +++ b/ckan/tests/legacy/logic/test_action.py @@ -6,7 +6,7 @@ from pprint import pprint from nose.tools import assert_equal, assert_raises from nose.plugins.skip import SkipTest -from pylons import config +from ckan.common import config import datetime import mock diff --git a/ckan/tests/legacy/logic/test_auth.py b/ckan/tests/legacy/logic/test_auth.py index f7b94a681c5..4544d8cab8b 100644 --- a/ckan/tests/legacy/logic/test_auth.py +++ b/ckan/tests/legacy/logic/test_auth.py @@ -1,7 +1,7 @@ # encoding: utf-8 import paste -from pylons import config +from ckan.common import config import ckan.config.middleware import ckan.tests.legacy as tests from ckan.logic import get_action diff --git a/ckan/tests/legacy/misc/test_mock_mail_server.py b/ckan/tests/legacy/misc/test_mock_mail_server.py index a0a1ae28a5d..6d657d821e1 100644 --- a/ckan/tests/legacy/misc/test_mock_mail_server.py +++ b/ckan/tests/legacy/misc/test_mock_mail_server.py @@ -2,7 +2,7 @@ import time from nose.tools import assert_equal -from pylons import config +from ckan.common import config from email.mime.text import MIMEText import hashlib diff --git a/ckan/tests/legacy/misc/test_sync.py b/ckan/tests/legacy/misc/test_sync.py index 10b355e47fb..7c7bf3669aa 100644 --- a/ckan/tests/legacy/misc/test_sync.py +++ b/ckan/tests/legacy/misc/test_sync.py @@ -5,7 +5,7 @@ import urllib2 import time -from pylons import config +from ckan.common import config import ckan.model as model from ckan.tests.legacy import * diff --git a/ckan/tests/legacy/mock_mail_server.py b/ckan/tests/legacy/mock_mail_server.py index 81db8c4c09a..501cdd1055c 100644 --- a/ckan/tests/legacy/mock_mail_server.py +++ b/ckan/tests/legacy/mock_mail_server.py @@ -5,7 +5,7 @@ import socket from smtpd import SMTPServer -from pylons import config +from ckan.common import config class MockSmtpServer(SMTPServer): diff --git a/ckan/tests/legacy/test_plugins.py b/ckan/tests/legacy/test_plugins.py index 15a0674735f..3af1524ad4b 100644 --- a/ckan/tests/legacy/test_plugins.py +++ b/ckan/tests/legacy/test_plugins.py @@ -6,7 +6,7 @@ from nose.tools import raises, assert_equal from unittest import TestCase from pyutilib.component.core import PluginGlobals -from pylons import config +from ckan.common import config import ckan.logic as logic import ckan.authz as authz diff --git a/ckan/tests/lib/search/test_index.py b/ckan/tests/lib/search/test_index.py index dbee706d983..fcf34694376 100644 --- a/ckan/tests/lib/search/test_index.py +++ b/ckan/tests/lib/search/test_index.py @@ -6,7 +6,7 @@ import nose from nose.tools import assert_equal, assert_in, assert_not_in -from pylons import config +from ckan.common import config import ckan.lib.search as search import ckan.tests.helpers as helpers diff --git a/ckan/tests/lib/test_datapreview.py b/ckan/tests/lib/test_datapreview.py index de80b55b5c3..dd4c85dc3df 100644 --- a/ckan/tests/lib/test_datapreview.py +++ b/ckan/tests/lib/test_datapreview.py @@ -1,7 +1,7 @@ # encoding: utf-8 import nose -from pylons import config +from ckan.common import config import ckan.plugins as p import ckan.lib.datapreview as datapreview diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index 143be8373b4..d04fa24d4f7 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -1,13 +1,13 @@ # encoding: utf-8 from nose.tools import assert_equal, assert_raises, assert_in -from pylons import config from email.mime.text import MIMEText from email.parser import Parser from email.header import decode_header import hashlib import base64 +from ckan.common import config import ckan.model as model import ckan.lib.helpers as h import ckan.lib.mailer as mailer diff --git a/ckan/tests/logic/action/test_create.py b/ckan/tests/logic/action/test_create.py index 37cbce676eb..41bad556f69 100644 --- a/ckan/tests/logic/action/test_create.py +++ b/ckan/tests/logic/action/test_create.py @@ -4,7 +4,7 @@ ''' -from pylons import config +from ckan.common import config import mock import nose.tools diff --git a/ckan/tests/logic/action/test_patch.py b/ckan/tests/logic/action/test_patch.py index e910b1618dc..4de8a8765c5 100644 --- a/ckan/tests/logic/action/test_patch.py +++ b/ckan/tests/logic/action/test_patch.py @@ -5,7 +5,7 @@ from nose.tools import assert_equals, assert_raises import mock -import pylons.config as config +from ckan.common import config from ckan.tests import helpers, factories diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py index 00813019e47..ce7d1e4c68e 100644 --- a/ckan/tests/logic/action/test_update.py +++ b/ckan/tests/logic/action/test_update.py @@ -5,7 +5,7 @@ import nose.tools import mock -import pylons.config as config +from ckan.common import config import ckan.logic as logic import ckan.lib.app_globals as app_globals diff --git a/ckan/tests/model/test_license.py b/ckan/tests/model/test_license.py index 50c9510c420..733cbdb5f4b 100644 --- a/ckan/tests/model/test_license.py +++ b/ckan/tests/model/test_license.py @@ -3,7 +3,7 @@ import os from nose.tools import assert_equal -from pylons import config +from ckan.common import config from ckan.model.license import LicenseRegister from ckan.tests import helpers diff --git a/ckanext/datapusher/logic/action.py b/ckanext/datapusher/logic/action.py index 44089dd9656..c246cd775e5 100644 --- a/ckanext/datapusher/logic/action.py +++ b/ckanext/datapusher/logic/action.py @@ -7,12 +7,12 @@ from dateutil.parser import parse as parse_date -import pylons import requests import ckan.lib.navl.dictization_functions import ckan.logic as logic import ckan.plugins as p +from ckan.common import config import ckanext.datapusher.logic.schema as dpschema import ckanext.datapusher.interfaces as interfaces @@ -57,9 +57,9 @@ def datapusher_submit(context, data_dict): except logic.NotFound: return False - datapusher_url = pylons.config.get('ckan.datapusher.url') + datapusher_url = config.get('ckan.datapusher.url') - site_url = pylons.config['ckan.site_url'] + site_url = config['ckan.site_url'] callback_url = site_url.rstrip('/') + '/api/3/action/datapusher_hook' user = p.toolkit.get_action('user_show')(context, {'id': context['user']}) @@ -252,7 +252,7 @@ def datapusher_status(context, data_dict): 'key': 'datapusher' }) - datapusher_url = pylons.config.get('ckan.datapusher.url') + datapusher_url = config.get('ckan.datapusher.url') if not datapusher_url: raise p.toolkit.ValidationError( {'configuration': ['ckan.datapusher.url not in config file']}) diff --git a/ckanext/datapusher/tests/test.py b/ckanext/datapusher/tests/test.py index 58802fb07ec..870361b514f 100644 --- a/ckanext/datapusher/tests/test.py +++ b/ckanext/datapusher/tests/test.py @@ -6,8 +6,6 @@ import nose import datetime -import pylons -from pylons import config import sqlalchemy.orm as orm import paste.fixture @@ -17,6 +15,8 @@ import ckan.tests.legacy as tests import ckan.config.middleware as middleware +from ckan.common import config + import ckanext.datastore.db as db from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type @@ -71,7 +71,7 @@ def setup_class(cls): cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) @@ -102,7 +102,7 @@ def test_create_ckan_resource_in_package(self): @httpretty.activate def test_providing_res_with_url_calls_datapusher_correctly(self): - pylons.config['datapusher.url'] = 'http://datapusher.ckan.org' + config['datapusher.url'] = 'http://datapusher.ckan.org' httpretty.HTTPretty.register_uri( httpretty.HTTPretty.POST, 'http://datapusher.ckan.org/job', @@ -128,7 +128,7 @@ def test_providing_res_with_url_calls_datapusher_correctly(self): @httpretty.activate def test_pass_the_received_ignore_hash_param_to_the_datapusher(self): - pylons.config['datapusher.url'] = 'http://datapusher.ckan.org' + config['datapusher.url'] = 'http://datapusher.ckan.org' httpretty.HTTPretty.register_uri( httpretty.HTTPretty.POST, 'http://datapusher.ckan.org/job', diff --git a/ckanext/datapusher/tests/test_interfaces.py b/ckanext/datapusher/tests/test_interfaces.py index d21bf3ab2b7..49b8efd7fad 100644 --- a/ckanext/datapusher/tests/test_interfaces.py +++ b/ckanext/datapusher/tests/test_interfaces.py @@ -7,7 +7,7 @@ import datetime from nose.tools import raises -from pylons import config +from ckan.common import config import sqlalchemy.orm as orm import paste.fixture diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 4e6b47956db..46d4cbbac9a 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -11,7 +11,6 @@ import copy import hashlib -import pylons import distutils.version import sqlalchemy from sqlalchemy.exc import (ProgrammingError, IntegrityError, @@ -22,7 +21,7 @@ import ckan.plugins.toolkit as toolkit import ckanext.datastore.interfaces as interfaces import ckanext.datastore.helpers as datastore_helpers -from ckan.common import OrderedDict +from ckan.common import OrderedDict, config log = logging.getLogger(__name__) @@ -104,9 +103,8 @@ def _get_engine(data_dict): engine = _engines.get(connection_url) if not engine: - import pylons extras = {'url': connection_url} - engine = sqlalchemy.engine_from_config(pylons.config, + engine = sqlalchemy.engine_from_config(config, 'ckan.datastore.sqlalchemy.', **extras) _engines[connection_url] = engine @@ -129,7 +127,7 @@ def _cache_types(context): native_json)) data_dict = { - 'connection_url': pylons.config['ckan.datastore.write_url']} + 'connection_url': config['ckan.datastore.write_url']} engine = _get_engine(data_dict) with engine.begin() as connection: connection.execute( @@ -471,7 +469,7 @@ def _build_fts_indexes(connection, data_dict, sql_index_str_method, fields): fts_indexes = [] resource_id = data_dict['resource_id'] # FIXME: This is repeated on the plugin.py, we should keep it DRY - default_fts_lang = pylons.config.get('ckan.datastore.default_fts_lang') + default_fts_lang = config.get('ckan.datastore.default_fts_lang') if default_fts_lang is None: default_fts_lang = u'english' fts_lang = data_dict.get('lang', default_fts_lang) @@ -506,7 +504,7 @@ def _generate_index_name(resource_id, field): def _get_fts_index_method(): - method = pylons.config.get('ckan.datastore.default_fts_index_method') + method = config.get('ckan.datastore.default_fts_index_method') return method or 'gist' @@ -1327,8 +1325,8 @@ def make_public(context, data_dict): def get_all_resources_ids_in_datastore(): - read_url = pylons.config.get('ckan.datastore.read_url') - write_url = pylons.config.get('ckan.datastore.write_url') + read_url = config.get('ckan.datastore.read_url') + write_url = config.get('ckan.datastore.write_url') data_dict = { 'connection_url': read_url or write_url } diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 9869d14baaa..77379c96a9a 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -2,13 +2,12 @@ import logging -import pylons import sqlalchemy -import ckan.lib.base as base import ckan.lib.navl.dictization_functions import ckan.logic as logic import ckan.plugins as p +from ckan.common import config import ckanext.datastore.db as db import ckanext.datastore.logic.schema as dsschema import ckanext.datastore.helpers as datastore_helpers @@ -122,7 +121,7 @@ def datastore_create(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] + data_dict['connection_url'] = config['ckan.datastore.write_url'] # validate aliases aliases = datastore_helpers.get_list(data_dict.get('aliases', [])) @@ -135,7 +134,7 @@ def datastore_create(context, data_dict): # create a private datastore resource, if necessary model = _get_or_bust(context, 'model') resource = model.Resource.get(data_dict['resource_id']) - legacy_mode = 'ckan.datastore.read_url' not in pylons.config + legacy_mode = 'ckan.datastore.read_url' not in config if not legacy_mode and resource.package.private: data_dict['private'] = True @@ -209,7 +208,7 @@ def datastore_upsert(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] + data_dict['connection_url'] = config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" @@ -251,7 +250,7 @@ def _type_lookup(t): resource_id = _get_or_bust(data_dict, 'id') resource = p.toolkit.get_action('resource_show')(context, {'id':resource_id}) - data_dict['connection_url'] = pylons.config['ckan.datastore.read_url'] + data_dict['connection_url'] = config['ckan.datastore.read_url'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" WHERE name = :id AND alias_of IS NULL''') @@ -336,7 +335,7 @@ def datastore_delete(context, data_dict): resource_id = data_dict['resource_id'] _check_read_only(context, resource_id) - data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] + data_dict['connection_url'] = config['ckan.datastore.write_url'] res_id = data_dict['resource_id'] resources_sql = sqlalchemy.text(u'''SELECT 1 FROM "_table_metadata" @@ -435,7 +434,7 @@ def datastore_search(context, data_dict): raise p.toolkit.ValidationError(errors) res_id = data_dict['resource_id'] - data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] + data_dict['connection_url'] = config['ckan.datastore.write_url'] resources_sql = sqlalchemy.text(u'''SELECT alias_of FROM "_table_metadata" WHERE name = :id''') @@ -499,7 +498,7 @@ def datastore_search_sql(context, data_dict): p.toolkit.check_access('datastore_search_sql', context, data_dict) - data_dict['connection_url'] = pylons.config['ckan.datastore.read_url'] + data_dict['connection_url'] = config['ckan.datastore.read_url'] result = db.search_sql(context, data_dict) result.pop('id', None) @@ -522,7 +521,7 @@ def datastore_make_private(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] + data_dict['connection_url'] = config['ckan.datastore.write_url'] if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( @@ -548,7 +547,7 @@ def datastore_make_public(context, data_dict): data_dict['resource_id'] = data_dict['id'] res_id = _get_or_bust(data_dict, 'resource_id') - data_dict['connection_url'] = pylons.config['ckan.datastore.write_url'] + data_dict['connection_url'] = config['ckan.datastore.write_url'] if not _resource_exists(context, data_dict): raise p.toolkit.ObjectNotFound(p.toolkit._( diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 5847b4302af..7d7fe15e16b 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -4,12 +4,12 @@ import logging import re -import pylons import sqlalchemy.engine.url as sa_url import ckan.plugins as p import ckan.logic as logic import ckan.model as model +from ckan.common import config import ckanext.datastore.logic.action as action import ckanext.datastore.logic.auth as auth import ckanext.datastore.db as db @@ -494,7 +494,7 @@ def _textsearch_query(self, data_dict): return statements_str, rank_columns_str def _fts_lang(self, lang=None): - default_fts_lang = pylons.config.get('ckan.datastore.default_fts_lang') + default_fts_lang = config.get('ckan.datastore.default_fts_lang') if default_fts_lang is None: default_fts_lang = u'english' return lang or default_fts_lang diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 8e97355b525..086a3c0b739 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -5,11 +5,10 @@ import sys from nose.tools import assert_equal, raises -import pylons -from pylons import config import sqlalchemy.orm as orm import paste.fixture +from ckan.common import config import ckan.plugins as p import ckan.lib.create_test_data as ctd import ckan.model as model @@ -181,7 +180,7 @@ def _get_index_names(self, resource_id): def _execute_sql(self, sql, *args): engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) session = orm.scoped_session(orm.sessionmaker(bind=engine)) return session.connection().execute(sql, *args) @@ -243,7 +242,7 @@ def setup_class(cls): cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index f0af7b861e9..53f9d487f06 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -3,7 +3,6 @@ import json import nose -import pylons import sqlalchemy import sqlalchemy.orm as orm @@ -11,6 +10,7 @@ import ckan.lib.create_test_data as ctd import ckan.model as model import ckan.tests.legacy as tests +from ckan.common import config import ckanext.datastore.db as db from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type @@ -43,7 +43,7 @@ def setup_class(cls): } engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) set_url_type( model.Package.get('annakarenina').resources, cls.sysadmin_user) diff --git a/ckanext/datastore/tests/test_disable.py b/ckanext/datastore/tests/test_disable.py index dbe925d5971..ad6912fe2cc 100644 --- a/ckanext/datastore/tests/test_disable.py +++ b/ckanext/datastore/tests/test_disable.py @@ -2,7 +2,7 @@ import nose -import pylons.config as config +from ckan.common import config import ckan.plugins as p import nose.tools as t diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 15e4b447238..b966bedf7fa 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -4,7 +4,7 @@ import nose from nose.tools import assert_equals -from pylons import config +from ckan.common import config import sqlalchemy.orm as orm import paste.fixture diff --git a/ckanext/datastore/tests/test_helpers.py b/ckanext/datastore/tests/test_helpers.py index 6a3e121bdad..9cc05c3dded 100644 --- a/ckanext/datastore/tests/test_helpers.py +++ b/ckanext/datastore/tests/test_helpers.py @@ -1,9 +1,9 @@ # encoding: utf-8 -import pylons import sqlalchemy.orm as orm import nose +from ckan.common import config import ckanext.datastore.helpers as datastore_helpers import ckanext.datastore.tests.helpers as datastore_test_helpers import ckanext.datastore.db as db @@ -67,11 +67,11 @@ class TestGetTables(object): @classmethod def setup_class(cls): - if not pylons.config.get('ckan.datastore.read_url'): + if not config.get('ckan.datastore.read_url'): raise nose.SkipTest('Datastore runs on legacy mode, skipping...') engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']} + {'connection_url': config['ckan.datastore.write_url']} ) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index 1ef119c320c..c472b52bdc3 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -4,7 +4,6 @@ import nose import pprint -import pylons import sqlalchemy.orm as orm import ckan.plugins as p @@ -12,6 +11,7 @@ import ckan.model as model import ckan.tests.legacy as tests +from ckan.common import config import ckanext.datastore.db as db from ckanext.datastore.tests.helpers import extract, rebuild_all_dbs @@ -166,7 +166,7 @@ def setup_class(cls): u'rating with %': u'99%'}] engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']} + {'connection_url': config['ckan.datastore.write_url']} ) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @@ -839,7 +839,7 @@ def setup_class(cls): cls.expected_join_results = [{u'first': 1, u'second': 1}, {u'first': 1, u'second': 2}] engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -942,7 +942,7 @@ def test_read_private(self): 'model': model} data_dict = { 'resource_id': self.data['resource_id'], - 'connection_url': pylons.config['ckan.datastore.write_url']} + 'connection_url': config['ckan.datastore.write_url']} p.toolkit.get_action('datastore_make_private')(context, data_dict) query = 'SELECT * FROM "{0}"'.format(self.data['resource_id']) data = {'sql': query} diff --git a/ckanext/datastore/tests/test_unit.py b/ckanext/datastore/tests/test_unit.py index 653fc9699f1..c02d30c60d6 100644 --- a/ckanext/datastore/tests/test_unit.py +++ b/ckanext/datastore/tests/test_unit.py @@ -1,11 +1,10 @@ # encoding: utf-8 import unittest -import pylons import nose import mock -from pylons import config +from ckan.common import config import ckan.tests.legacy as tests import ckanext.datastore.db as db @@ -37,7 +36,7 @@ def test_pg_version_check(self): if not tests.is_datastore_supported(): raise nose.SkipTest("Datastore not supported") engine = db._get_engine( - {'connection_url': pylons.config['sqlalchemy.url']}) + {'connection_url': config['sqlalchemy.url']}) connection = engine.connect() assert db._pg_version_is_at_least(connection, '8.0') assert not db._pg_version_is_at_least(connection, '10.0') diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index 22fedccab35..b7a09bf7dae 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -4,7 +4,6 @@ import nose import datetime -import pylons import sqlalchemy.orm as orm import ckan.plugins as p @@ -14,6 +13,7 @@ import ckan.tests.helpers as helpers import ckan.tests.factories as factories +from ckan.common import config import ckanext.datastore.db as db from ckanext.datastore.tests.helpers import rebuild_all_dbs, set_url_type @@ -114,7 +114,7 @@ def setup_class(cls): assert res_dict['success'] is True engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -366,7 +366,7 @@ def setup_class(cls): assert res_dict['success'] is True engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod @@ -473,7 +473,7 @@ def setup_class(cls): assert res_dict['success'] is True engine = db._get_engine( - {'connection_url': pylons.config['ckan.datastore.write_url']}) + {'connection_url': config['ckan.datastore.write_url']}) cls.Session = orm.scoped_session(orm.sessionmaker(bind=engine)) @classmethod diff --git a/ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py b/ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py index f32cba56dbc..e7eb60fa47d 100644 --- a/ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py +++ b/ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py @@ -1,6 +1,6 @@ # encoding: utf-8 -import pylons.config as config +from ckan.common import config import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit diff --git a/ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py b/ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py index aca2f50612c..6ee8416ccf6 100644 --- a/ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py +++ b/ckanext/example_iauthfunctions/plugin_v6_parent_auth_functions.py @@ -1,6 +1,6 @@ # encoding: utf-8 -import pylons.config as config +from ckan.common import config import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit diff --git a/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py b/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py index d2a482049c1..f5bafc15bb6 100644 --- a/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py +++ b/ckanext/example_iauthfunctions/tests/test_example_iauthfunctions.py @@ -5,7 +5,6 @@ ''' import paste.fixture import pylons.test -import pylons.config as config import webtest from nose.tools import assert_raises @@ -16,6 +15,7 @@ import ckan.plugins import ckan.tests.factories as factories import ckan.logic as logic +from ckan.common import config class TestExampleIAuthFunctionsPluginV6ParentAuthFunctions(object): @@ -89,7 +89,7 @@ class TestExampleIAuthFunctionsCustomConfigSetting(object): ''' def _get_app(self, users_can_create_groups): - # Set the custom config option in pylons.config. + # Set the custom config option in config. config['ckan.iauthfunctions.users_can_create_groups'] = ( users_can_create_groups) @@ -103,7 +103,7 @@ def _get_app(self, users_can_create_groups): def teardown(self): - # Remove the custom config option from pylons.config. + # Remove the custom config option from config. del config['ckan.iauthfunctions.users_can_create_groups'] # Delete any stuff that's been created in the db, so it doesn't diff --git a/ckanext/example_iconfigurer/tests/test_iconfigurer_update_config.py b/ckanext/example_iconfigurer/tests/test_iconfigurer_update_config.py index f5f44b957f9..5f358488227 100644 --- a/ckanext/example_iconfigurer/tests/test_iconfigurer_update_config.py +++ b/ckanext/example_iconfigurer/tests/test_iconfigurer_update_config.py @@ -2,7 +2,7 @@ import nose.tools -from pylons import config +from ckan.common import config import ckan.lib.app_globals as app_globals diff --git a/ckanext/example_idatasetform/tests/test_example_idatasetform.py b/ckanext/example_idatasetform/tests/test_example_idatasetform.py index 5952ba0f54d..42e0e85c5e4 100644 --- a/ckanext/example_idatasetform/tests/test_example_idatasetform.py +++ b/ckanext/example_idatasetform/tests/test_example_idatasetform.py @@ -2,7 +2,7 @@ import nose.tools as nt -import pylons.config as config +from ckan.common import config import ckan.model as model import ckan.plugins as plugins diff --git a/ckanext/example_iresourcecontroller/tests/test_example_iresourcecontroller.py b/ckanext/example_iresourcecontroller/tests/test_example_iresourcecontroller.py index 3481ded4e40..b4ccf746b02 100644 --- a/ckanext/example_iresourcecontroller/tests/test_example_iresourcecontroller.py +++ b/ckanext/example_iresourcecontroller/tests/test_example_iresourcecontroller.py @@ -3,7 +3,7 @@ '''Tests for the ckanext.example_iauthfunctions extension. ''' -import pylons.config as config +from ckan.common import config import webtest import ckan.model as model diff --git a/ckanext/example_iuploader/test/test_plugin.py b/ckanext/example_iuploader/test/test_plugin.py index 08daf634d21..3259fa9ec6c 100644 --- a/ckanext/example_iuploader/test/test_plugin.py +++ b/ckanext/example_iuploader/test/test_plugin.py @@ -10,12 +10,12 @@ assert_is_instance ) from pyfakefs import fake_filesystem -from pylons import config from routes import url_for import ckan.lib.uploader import ckan.model as model import ckan.plugins as plugins +from ckan.common import config import ckan.tests.factories as factories import ckan.tests.helpers as helpers import ckanext.example_iuploader.plugin as plugin diff --git a/ckanext/example_ivalidators/tests/test_ivalidators.py b/ckanext/example_ivalidators/tests/test_ivalidators.py index 47f64103da6..ca8d292397f 100644 --- a/ckanext/example_ivalidators/tests/test_ivalidators.py +++ b/ckanext/example_ivalidators/tests/test_ivalidators.py @@ -1,7 +1,7 @@ # encoding: utf-8 from nose.tools import assert_equals, assert_raises -import pylons.config as config +from ckan.common import config from ckan.plugins.toolkit import get_validator, Invalid from ckan import plugins diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py index 5c99321d793..9a519b40a2f 100644 --- a/ckanext/multilingual/plugin.py +++ b/ckanext/multilingual/plugin.py @@ -3,9 +3,9 @@ import ckan from ckan.plugins import SingletonPlugin, implements, IPackageController from ckan.plugins import IGroupController, IOrganizationController, ITagController, IResourceController -import pylons + +from ckan.common import request, config, c from ckan.logic import get_action -from pylons import config LANGS = ['en', 'fr', 'de', 'es', 'it', 'nl', 'ro', 'pt', 'pl'] @@ -14,8 +14,8 @@ def translate_data_dict(data_dict): as possible translated into the desired or the fallback language. ''' - desired_lang_code = pylons.request.environ['CKAN_LANG'] - fallback_lang_code = pylons.config.get('ckan.locale_default', 'en') + desired_lang_code = request.environ['CKAN_LANG'] + fallback_lang_code = config.get('ckan.locale_default', 'en') # Get a flattened copy of data_dict to do the translation on. flattened = ckan.lib.navl.dictization_functions.flatten_dict( @@ -109,8 +109,8 @@ def translate_resource_data_dict(data_dict): as possible translated into the desired or the fallback language. ''' - desired_lang_code = pylons.request.environ['CKAN_LANG'] - fallback_lang_code = pylons.config.get('ckan.locale_default', 'en') + desired_lang_code = request.environ['CKAN_LANG'] + fallback_lang_code = config.get('ckan.locale_default', 'en') # Get a flattened copy of data_dict to do the translation on. flattened = ckan.lib.navl.dictization_functions.flatten_dict( @@ -199,13 +199,13 @@ class MultilingualDataset(SingletonPlugin): def before_index(self, search_data): default_lang = search_data.get( - 'lang_code', - pylons.config.get('ckan.locale_default', 'en') + 'lang_code', + config.get('ckan.locale_default', 'en') ) ## translate title title = search_data.get('title') - search_data['title_' + default_lang] = title + search_data['title_' + default_lang] = title title_translations = get_action('term_translation_show')( {'model': ckan.model}, {'terms': [title], @@ -232,7 +232,7 @@ def before_index(self, search_data): 'lang_codes': LANGS}) text_field_items = dict(('text_' + lang, []) for lang in LANGS) - + text_field_items['text_' + default_lang].extend(all_terms) for translation in sorted(field_translations): @@ -241,14 +241,14 @@ def before_index(self, search_data): for key, value in text_field_items.iteritems(): search_data[key] = ' '.join(value) - + return search_data def before_search(self, search_params): lang_set = set(LANGS) try: - current_lang = pylons.request.environ['CKAN_LANG'] + current_lang = request.environ['CKAN_LANG'] except TypeError as err: if err.message == ('No object (name: request) has been registered ' 'for this thread'): @@ -284,8 +284,8 @@ def after_search(self, search_results, search_params): if not facets: return search_results - desired_lang_code = pylons.request.environ['CKAN_LANG'] - fallback_lang_code = pylons.config.get('ckan.locale_default', 'en') + desired_lang_code = request.environ['CKAN_LANG'] + fallback_lang_code = config.get('ckan.locale_default', 'en') # Look up translations for all of the facets in one db query. terms = set() @@ -323,9 +323,8 @@ def before_view(self, dataset_dict): # all the terms in c.fields (c.fields contains the selected facets) # and save them in c.translated_fields where the templates can # retrieve them later. - c = pylons.c - desired_lang_code = pylons.request.environ['CKAN_LANG'] - fallback_lang_code = pylons.config.get('ckan.locale_default', 'en') + desired_lang_code = request.environ['CKAN_LANG'] + fallback_lang_code = config.get('ckan.locale_default', 'en') terms = [value for param, value in c.fields] translations = get_action('term_translation_show')( {'model': ckan.model}, diff --git a/ckanext/reclineview/tests/test_view.py b/ckanext/reclineview/tests/test_view.py index 897be8048f2..0cbd353ec42 100644 --- a/ckanext/reclineview/tests/test_view.py +++ b/ckanext/reclineview/tests/test_view.py @@ -1,7 +1,7 @@ # encoding: utf-8 import paste.fixture -import pylons.config as config +from ckan.common import config import ckan.model as model import ckan.tests.legacy as tests diff --git a/ckanext/resourceproxy/controller.py b/ckanext/resourceproxy/controller.py index 7d911187f8d..0020ea946a8 100644 --- a/ckanext/resourceproxy/controller.py +++ b/ckanext/resourceproxy/controller.py @@ -5,7 +5,7 @@ import requests -import pylons.config as config +from ckan.common import config import ckan.logic as logic import ckan.lib.base as base diff --git a/ckanext/resourceproxy/plugin.py b/ckanext/resourceproxy/plugin.py index 4cd1b5edd82..edfd34aa8a8 100644 --- a/ckanext/resourceproxy/plugin.py +++ b/ckanext/resourceproxy/plugin.py @@ -6,7 +6,7 @@ import ckan.plugins as p import ckan.lib.datapreview as datapreview import urlparse -from pylons import config +from ckan.common import config log = getLogger(__name__) diff --git a/ckanext/resourceproxy/tests/test_proxy.py b/ckanext/resourceproxy/tests/test_proxy.py index 78413da7685..2d67d1a9303 100644 --- a/ckanext/resourceproxy/tests/test_proxy.py +++ b/ckanext/resourceproxy/tests/test_proxy.py @@ -7,7 +7,7 @@ import nose import paste.fixture -from pylons import config +from ckan.common import config import ckan.model as model import ckan.tests.legacy as tests diff --git a/ckanext/stats/stats.py b/ckanext/stats/stats.py index 657cd2ba1f4..9a2f6418567 100644 --- a/ckanext/stats/stats.py +++ b/ckanext/stats/stats.py @@ -2,7 +2,7 @@ import datetime -from pylons import config +from ckan.common import config from sqlalchemy import Table, select, join, func, and_ import ckan.plugins as p diff --git a/ckanext/stats/tests/__init__.py b/ckanext/stats/tests/__init__.py index 2530688d211..d149dbb11b5 100644 --- a/ckanext/stats/tests/__init__.py +++ b/ckanext/stats/tests/__init__.py @@ -1,7 +1,7 @@ # encoding: utf-8 import paste.fixture -from pylons import config +from ckan.common import config from ckan.config.middleware import make_app class StatsFixture(object): diff --git a/ckanext/stats/tests/test_stats_plugin.py b/ckanext/stats/tests/test_stats_plugin.py index b87af1e4625..ab802917f73 100644 --- a/ckanext/stats/tests/test_stats_plugin.py +++ b/ckanext/stats/tests/test_stats_plugin.py @@ -9,7 +9,7 @@ class TestStatsPlugin(StatsFixture): def test_01_config(self): - from pylons import config + from ckan.common import config paths = config['extra_public_paths'] publicdir = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'public') diff --git a/ckanext/textview/tests/test_view.py b/ckanext/textview/tests/test_view.py index 1e6f9144cbe..7f3e4cfa7b9 100644 --- a/ckanext/textview/tests/test_view.py +++ b/ckanext/textview/tests/test_view.py @@ -1,7 +1,7 @@ # encoding: utf-8 import paste.fixture -import pylons.config as config +from ckan.common import config import urlparse import ckan.model as model diff --git a/doc/extensions/custom-config-settings.rst b/doc/extensions/custom-config-settings.rst index de0f0d264b2..81c350fdc1c 100644 --- a/doc/extensions/custom-config-settings.rst +++ b/doc/extensions/custom-config-settings.rst @@ -13,7 +13,7 @@ allows users to create new groups if a new config setting .. literalinclude:: ../../ckanext/example_iauthfunctions/plugin_v5_custom_config_setting.py The ``group_create`` authorization function in this plugin uses -:py:obj:`pylons.config` to read the setting from the config file, then calls +:py:obj:`config` to read the setting from the config file, then calls :py:func:`ckan.plugins.toolkit.asbool` to convert the value from a string (all config settings values are strings, when read from the file) to a boolean. From 37ecbe8bd44dc0dc03e05d82bf09048c7d1feed6 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 13 Jul 2016 12:02:34 +0100 Subject: [PATCH 100/210] [#2842] Add config to the plugins toolkit --- ckan/plugins/toolkit.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 427fc489fbb..53938a38c54 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -18,6 +18,8 @@ class _Toolkit(object): # contents should describe the available functions/objects. We check # that this list matches the actual availables in the initialisation contents = [ + # Global CKAN configuration object + 'config', # i18n translation '_', # i18n translation (plural form) @@ -144,6 +146,14 @@ def _initialize(self): t = self._toolkit # imported functions + t['config'] = common.config + self.docstring_overrides['config'] = '''The CKAN configuration object. + +It stores the configuration values defined in the :ref:`config_file`, eg:: + + title = toolkit.config.get("ckan.site_title") + +''' t['_'] = common._ self.docstring_overrides['_'] = '''The Pylons ``_()`` function. From aa396e274f1417b16f67c83c415f47d864d33b16 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 13 Jul 2016 14:24:18 +0100 Subject: [PATCH 101/210] [#2842] Restore Pylons config on config clear test --- ckan/tests/test_common.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ckan/tests/test_common.py b/ckan/tests/test_common.py index 6ff25c416b9..cf3dda812a2 100644 --- a/ckan/tests/test_common.py +++ b/ckan/tests/test_common.py @@ -54,6 +54,9 @@ def test_keys_works(self): def test_clear_works(self): + # Keep a copy of the original Pylons config + _original_pylons_config = pylons.config.copy() + my_conf = CKANConfig() my_conf[u'test_key_1'] = u'Test value 1' @@ -65,6 +68,9 @@ def test_clear_works(self): eq_(len(my_conf.keys()), 0) + # Restore Pylons config + pylons.config.update(_original_pylons_config) + def test_for_in_works(self): my_conf = CKANConfig() From a476eb2186c212417ba3f2281ec7c2856557ae1b Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 14 Jul 2016 13:59:47 +0100 Subject: [PATCH 102/210] [#3120] Format abort message with key --- ckan/lib/helpers.py | 3 ++- ckan/tests/controllers/test_feed.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index a19ade3132c..620f671dd98 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -1097,7 +1097,8 @@ def get_page_number(params, key='page', default=1): raise ValueError("Negative number not allowed") except ValueError: import ckan.lib.base as base - base.abort(400, ('"{key}" parameter must be a positive integer')) + base.abort(400, ('"{key}" parameter must be a positive integer' + .format(key=key))) return p diff --git a/ckan/tests/controllers/test_feed.py b/ckan/tests/controllers/test_feed.py index 57adcade8ea..da4500cd4c0 100644 --- a/ckan/tests/controllers/test_feed.py +++ b/ckan/tests/controllers/test_feed.py @@ -14,7 +14,7 @@ def test_atom_feed_page_zero_gives_error(self): id=group['name']) + '?page=0' app = self._get_test_app() res = app.get(offset, status=400) - assert '"{key}" parameter must be a positive integer' in res, res + assert '"page" parameter must be a positive integer' in res, res def test_atom_feed_page_negative_gives_error(self): group = factories.Group() @@ -22,7 +22,7 @@ def test_atom_feed_page_negative_gives_error(self): id=group['name']) + '?page=-2' app = self._get_test_app() res = app.get(offset, status=400) - assert '"{key}" parameter must be a positive integer' in res, res + assert '"page" parameter must be a positive integer' in res, res def test_atom_feed_page_not_int_gives_error(self): group = factories.Group() @@ -30,4 +30,4 @@ def test_atom_feed_page_not_int_gives_error(self): id=group['name']) + '?page=abc' app = self._get_test_app() res = app.get(offset, status=400) - assert '"{key}" parameter must be a positive integer' in res, res + assert '"page" parameter must be a positive integer' in res, res From bfd71bfdfd142c81be15a989e6a110febefe92f4 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 14 Jul 2016 16:21:24 +0100 Subject: [PATCH 103/210] [#2842] Remove debug command --- ckan/plugins/toolkit.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py index 53938a38c54..899763806ea 100644 --- a/ckan/plugins/toolkit.py +++ b/ckan/plugins/toolkit.py @@ -366,7 +366,6 @@ def _add_ckan_admin_tabs(cls, config, route_name, tab_label, # update the config with the updated admin_tabs dict config.update({config_var: admin_tabs_dict}) -# import ipdb; ipdb.set_trace() @classmethod def _version_str_2_list(cls, v_str): ''' convert a version string into a list of ints From 8b3bb459aeae16f4760582f7caeef8f663698098 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 15 Jul 2016 15:27:13 +0100 Subject: [PATCH 104/210] [#3167] Fix q parameter on followee_list The data_dict for followee_list was validated using default_follow_user_schema, which didn't have validators for q, so the data_dict ended up like: {'__extras': {'q': u'Environment}, 'id': u'6cdc2a4c-2814-4a29-98f4-'} Added q to the schema and a couple of tests --- ckan/logic/schema.py | 3 +- ckan/tests/logic/action/test_get.py | 43 +++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f6b369a26f3..78ac4ed5a40 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -522,7 +522,8 @@ def default_create_activity_schema(): def default_follow_user_schema(): schema = {'id': [not_missing, not_empty, unicode, - convert_user_name_or_id_to_id]} + convert_user_name_or_id_to_id], + 'q': [ignore_missing]} return schema diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index c664d817b18..54a1fd2ab70 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -2018,3 +2018,46 @@ def test_user_delete_marks_membership_of_org_as_deleted(self): capacity='member') eq(len(org_members), 0) + + +class TestFollow(helpers.FunctionalTestBase): + + def test_followee_list(self): + + group1 = factories.Group(title='Finance') + group2 = factories.Group(title='Environment') + group3 = factories.Group(title='Education') + + user = factories.User() + + context = {'user': user['name']} + + helpers.call_action('follow_group', context, id=group1['id']) + helpers.call_action('follow_group', context, id=group2['id']) + + followee_list = helpers.call_action('followee_list', context, + id=user['name']) + + eq(len(followee_list), 2) + eq(sorted([f['display_name'] for f in followee_list]), + ['Environment', 'Finance']) + + def test_followee_list_with_q(self): + + group1 = factories.Group(title='Finance') + group2 = factories.Group(title='Environment') + group3 = factories.Group(title='Education') + + user = factories.User() + + context = {'user': user['name']} + + helpers.call_action('follow_group', context, id=group1['id']) + helpers.call_action('follow_group', context, id=group2['id']) + + followee_list = helpers.call_action('followee_list', context, + id=user['name'], + q='E') + + eq(len(followee_list), 1) + eq(followee_list[0]['display_name'], 'Environment') From f754749be6a9c661be984fe6f0470880b0baa8c9 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 21 Jul 2016 15:24:52 +0000 Subject: [PATCH 105/210] Docs, spelling, pep8 --- ckan/lib/navl/dictization_functions.py | 58 ++++++++++++++++++-------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py index 5461a74eb92..d79b9f4745f 100644 --- a/ckan/lib/navl/dictization_functions.py +++ b/ckan/lib/navl/dictization_functions.py @@ -62,8 +62,24 @@ def flattened_order_key(key): return tuple([len(key)] + list(key)) def flatten_schema(schema, flattened=None, key=None): - '''convert schema into flat dict where the keys are tuples''' + '''convert schema into flat dict, where the keys become tuples + e.g. + { + "toplevel": [validators], + "parent": { + "child1": [validators], + "child2": [validators], + } + } + becomes: + { + ('toplevel',): [validators], + ('parent', 'child1'): [validators], + ('parent', 'child2'): [validators], + } + See also: test_flatten_schema() + ''' flattened = flattened or {} old_key = key or [] @@ -76,21 +92,21 @@ def flatten_schema(schema, flattened=None, key=None): return flattened -def get_all_key_combinations(data, flattented_schema): +def get_all_key_combinations(data, flattened_schema): '''Compare the schema against the given data and get all valid tuples that match the schema ignoring the last value in the tuple. ''' - schema_prefixes = set([key[:-1] for key in flattented_schema]) + schema_prefixes = set([key[:-1] for key in flattened_schema]) combinations = set([()]) for key in sorted(data.keys(), key=flattened_order_key): - ## make sure the tuple key is a valid one in the schema + # make sure the tuple key is a valid one in the schema key_prefix = key[:-1:2] if key_prefix not in schema_prefixes: continue - ## make sure the parent key exists, this is assured by sorting the keys - ## first + # make sure the parent key exists, this is assured by sorting the keys + # first if tuple(tuple(key[:-3])) not in combinations: continue combinations.add(tuple(key[:-1])) @@ -101,9 +117,9 @@ def make_full_schema(data, schema): '''make schema by getting all valid combinations and making sure that all keys are available''' - flattented_schema = flatten_schema(schema) + flattened_schema = flatten_schema(schema) - key_combinations = get_all_key_combinations(data, flattented_schema) + key_combinations = get_all_key_combinations(data, flattened_schema) full_schema = {} @@ -119,27 +135,35 @@ def make_full_schema(data, schema): return full_schema def augment_data(data, schema): - '''add missing, extras and junk data''' - flattented_schema = flatten_schema(schema) - key_combinations = get_all_key_combinations(data, flattented_schema) + '''Takes 'flattened' data, compares it with the schema, and returns it with + any problems marked, as follows: + + * keys in the data not in the schema are moved into a list under new key + ('__junk') + * keys in the schema but not data are added as keys with value 'missing' + + ''' + flattened_schema = flatten_schema(schema) + key_combinations = get_all_key_combinations(data, flattened_schema) full_schema = make_full_schema(data, schema) new_data = copy.copy(data) - ## fill junk and extras + # fill junk and extras for key, value in new_data.items(): if key in full_schema: continue - ## check if any thing naugthy is placed against subschemas + # check if any thing naughty is placed against subschemas initial_tuple = key[::2] if initial_tuple in [initial_key[:len(initial_tuple)] - for initial_key in flattented_schema]: - if data[key] <> []: + for initial_key in flattened_schema]: + if data[key] != []: raise DataError('Only lists of dicts can be placed against ' - 'subschema %s, not %s' % (key,type(data[key]))) + 'subschema %s, not %s' % + (key, type(data[key]))) if key[:-1] in key_combinations: extras_key = key[:-1] + ('__extras',) @@ -152,7 +176,7 @@ def augment_data(data, schema): new_data[("__junk",)] = junk new_data.pop(key) - ## add missing + # add missing for key, value in full_schema.items(): if key not in new_data and not key[-1].startswith("__"): From 40cf640397dae3548b228082194c5e4ce7e62d17 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 22 Jul 2016 10:54:47 +0000 Subject: [PATCH 106/210] Completed pep8 of dictization_functions. Moved validate_flattened into test as its only used there (I checked all projects on github too). --- ckan/lib/navl/dictization_functions.py | 75 +++++++++++++--------- ckan/tests/legacy/lib/test_navl.py | 11 +++- ckan/tests/legacy/test_coding_standards.py | 3 +- 3 files changed, 55 insertions(+), 34 deletions(-) diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py index d79b9f4745f..ff4bae72cd5 100644 --- a/ckan/lib/navl/dictization_functions.py +++ b/ckan/lib/navl/dictization_functions.py @@ -12,34 +12,46 @@ class Missing(object): def __unicode__(self): raise Invalid(_('Missing value')) + def __str__(self): raise Invalid(_('Missing value')) + def __int__(self): raise Invalid(_('Missing value')) + def __complex__(self): raise Invalid(_('Missing value')) + def __long__(self): raise Invalid(_('Missing value')) + def __float__(self): raise Invalid(_('Missing value')) + def __oct__(self): raise Invalid(_('Missing value')) + def __hex__(self): raise Invalid(_('Missing value')) + def __nonzero__(self): return False + missing = Missing() + class State(object): pass + class DictizationError(Exception): def __str__(self): if hasattr(self, 'error') and self.error: return repr(self.error) return '' + class Invalid(DictizationError): '''Exception raised by some validator, converter and dictization functions when the given value is invalid. @@ -48,19 +60,23 @@ class Invalid(DictizationError): def __init__(self, error, key=None): self.error = error + class DataError(DictizationError): def __init__(self, error): self.error = error + class StopOnError(DictizationError): '''error to stop validations for a particualar key''' pass + def flattened_order_key(key): '''order by key length first then values''' return tuple([len(key)] + list(key)) + def flatten_schema(schema, flattened=None, key=None): '''convert schema into flat dict, where the keys become tuples @@ -92,6 +108,7 @@ def flatten_schema(schema, flattened=None, key=None): return flattened + def get_all_key_combinations(data, flattened_schema): '''Compare the schema against the given data and get all valid tuples that match the schema ignoring the last value in the tuple. @@ -113,9 +130,10 @@ def get_all_key_combinations(data, flattened_schema): return combinations + def make_full_schema(data, schema): - '''make schema by getting all valid combinations and making sure that all keys - are available''' + '''make schema by getting all valid combinations and making sure that all + keys are available''' flattened_schema = flatten_schema(schema) @@ -134,6 +152,7 @@ def make_full_schema(data, schema): return full_schema + def augment_data(data, schema): '''Takes 'flattened' data, compares it with the schema, and returns it with any problems marked, as follows: @@ -184,6 +203,7 @@ def augment_data(data, schema): return new_data + def convert(converter, key, converted_data, errors, context): if inspect.isclass(converter) and issubclass(converter, fe.Validator): @@ -207,9 +227,9 @@ def convert(converter, key, converted_data, errors, context): converted_data[key] = value return except TypeError, e: - ## hack to make sure the type error was caused by the wrong - ## number of arguments given. - if not converter.__name__ in str(e): + # hack to make sure the type error was caused by the wrong + # number of arguments given. + if converter.__name__ not in str(e): raise except Invalid, e: errors[key].append(e.error) @@ -222,9 +242,9 @@ def convert(converter, key, converted_data, errors, context): errors[key].append(e.error) return except TypeError, e: - ## hack to make sure the type error was caused by the wrong - ## number of arguments given. - if not converter.__name__ in str(e): + # hack to make sure the type error was caused by the wrong + # number of arguments given. + if converter.__name__ not in str(e): raise try: @@ -235,6 +255,7 @@ def convert(converter, key, converted_data, errors, context): errors[key].append(e.error) return + def _remove_blank_keys(schema): for key, value in schema.items(): @@ -246,6 +267,7 @@ def _remove_blank_keys(schema): return schema + def validate(data, schema, context=None): '''Validate an unflattened nested dict against a schema.''' context = context or {} @@ -274,7 +296,7 @@ def validate(data, schema, context=None): errors_unflattened = unflatten(errors) - ##remove validators that passed + # remove validators that passed dicts_to_process = [errors_unflattened] while dicts_to_process: dict_to_process = dicts_to_process.pop() @@ -289,18 +311,6 @@ def validate(data, schema, context=None): return converted_data, errors_unflattened -def validate_flattened(data, schema, context=None): - - context = context or {} - assert isinstance(data, dict) - converted_data, errors = _validate(data, schema, context) - - for key, value in errors.items(): - if not value: - errors.pop(key) - - return converted_data, errors - def _validate(data, schema, context): '''validate a flattened dict against a schema''' @@ -309,7 +319,7 @@ def _validate(data, schema, context): errors = dict((key, []) for key in full_schema) - ## before run + # before run for key in sorted(full_schema, key=flattened_order_key): if key[-1] == '__before': for converter in full_schema[key]: @@ -318,7 +328,7 @@ def _validate(data, schema, context): except StopOnError: break - ## main run + # main run for key in sorted(full_schema, key=flattened_order_key): if not key[-1].startswith('__'): for converter in full_schema[key]: @@ -327,7 +337,7 @@ def _validate(data, schema, context): except StopOnError: break - ## extras run + # extras run for key in sorted(full_schema, key=flattened_order_key): if key[-1] == '__extras': for converter in full_schema[key]: @@ -336,7 +346,7 @@ def _validate(data, schema, context): except StopOnError: break - ## after run + # after run for key in reversed(sorted(full_schema, key=flattened_order_key)): if key[-1] == '__after': for converter in full_schema[key]: @@ -345,11 +355,12 @@ def _validate(data, schema, context): except StopOnError: break - ## junk + # junk if ('__junk',) in full_schema: for converter in full_schema[('__junk',)]: try: - convert(converter, ('__junk',), converted_data, errors, context) + convert(converter, ('__junk',), converted_data, errors, + context) except StopOnError: break @@ -370,8 +381,9 @@ def flatten_list(data, flattened=None, old_key=None): return flattened + def flatten_dict(data, flattened=None, old_key=None): - '''flatten a dict''' + '''Flatten a dict''' flattened = flattened or {} old_key = old_key or [] @@ -421,8 +433,8 @@ def unflatten(data): current_pos = unflattened if (len(flattend_key) > 1 - and not flattend_key[0] in convert_to_list - and not flattend_key[0] in unflattened): + and not flattend_key[0] in convert_to_list + and not flattend_key[0] in unflattened): convert_to_list.append(flattend_key[0]) for key in flattend_key[:-1]: @@ -435,7 +447,8 @@ def unflatten(data): current_pos[flattend_key[-1]] = data[flattend_key] for key in convert_to_list: - unflattened[key] = [unflattened[key][s] for s in sorted(unflattened[key])] + unflattened[key] = [unflattened[key][s] + for s in sorted(unflattened[key])] return unflattened diff --git a/ckan/tests/legacy/lib/test_navl.py b/ckan/tests/legacy/lib/test_navl.py index 04cfe17e158..8eb44c4e6a3 100644 --- a/ckan/tests/legacy/lib/test_navl.py +++ b/ckan/tests/legacy/lib/test_navl.py @@ -8,7 +8,7 @@ missing, augment_data, validate, - validate_flattened) + _validate) from pprint import pprint, pformat from ckan.lib.navl.validators import (identity_converter, empty, @@ -352,5 +352,14 @@ def test_range_validator(): assert errors == {'name': [u'Missing value'], 'email': [u'Please enter a number that is 10 or smaller']}, errors +def validate_flattened(data, schema, context=None): + context = context or {} + assert isinstance(data, dict) + converted_data, errors = _validate(data, schema, context) + for key, value in errors.items(): + if not value: + errors.pop(key) + + return converted_data, errors diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py index b9a5f0d3142..4f592b94cb5 100644 --- a/ckan/tests/legacy/test_coding_standards.py +++ b/ckan/tests/legacy/test_coding_standards.py @@ -14,7 +14,7 @@ Please do not add new files to the list as any new files should meet the current coding standards. Please add comments by files that fail if there -are legitamate reasons for the failure. +are legitimate reasons for the failure. ''' import sys @@ -398,7 +398,6 @@ class TestPep8(object): 'ckan/lib/jinja_extensions.py', 'ckan/lib/jsonp.py', 'ckan/lib/maintain.py', - 'ckan/lib/navl/dictization_functions.py', 'ckan/lib/navl/validators.py', 'ckan/lib/package_saver.py', 'ckan/lib/plugins.py', From bac4748ffdf7dfbfa4020e56e61706d78c3fb66a Mon Sep 17 00:00:00 2001 From: David Read Date: Mon, 25 Jul 2016 12:07:45 +0000 Subject: [PATCH 107/210] Fix - better exception for when bad smtp settings. --- ckan/lib/mailer.py | 9 ++++++++- ckan/tests/lib/test_mailer.py | 12 +++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index 1efa7a3bd3a..9eaf2fd6813 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -1,6 +1,7 @@ # encoding: utf-8 import smtplib +import socket import logging import uuid from time import time @@ -56,7 +57,13 @@ def _mail_recipient(recipient_name, recipient_email, config.get('smtp.starttls')) smtp_user = config.get('smtp.user') smtp_password = config.get('smtp.password') - smtp_connection.connect(smtp_server) + + try: + smtp_connection.connect(smtp_server) + except socket.error, e: + log.exception(e) + raise MailerException('SMTP server could not be connected to: "%s" %s' + % (smtp_server, e)) try: # Identify ourselves and prompt the server for supported features. smtp_connection.ehlo() diff --git a/ckan/tests/lib/test_mailer.py b/ckan/tests/lib/test_mailer.py index d04fa24d4f7..2ea839371cb 100644 --- a/ckan/tests/lib/test_mailer.py +++ b/ckan/tests/lib/test_mailer.py @@ -1,6 +1,6 @@ # encoding: utf-8 -from nose.tools import assert_equal, assert_raises, assert_in +from nose.tools import assert_equal, assert_raises, assert_in, assert_raises from email.mime.text import MIMEText from email.parser import Parser from email.header import decode_header @@ -223,3 +223,13 @@ def test_send_invite_email_with_org(self): body = self.get_email_body(msg[3]) assert_in(org['title'], body) assert_in(h.roles_translated()[role], body) + + @helpers.change_config('smtp.test_server', '999.999.999.999') + def test_bad_smtp_host(self): + test_email = {'recipient_name': 'Bob', + 'recipient_email': 'b@example.com', + 'subject': 'Meeting', + 'body': 'The meeting is cancelled.', + 'headers': {'header1': 'value1'}} + assert_raises(mailer.MailerException, + mailer.mail_recipient, **test_email) From 511083fdab263d8c1cfd53d1ad20b966f88a196d Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 3 Aug 2016 12:43:51 +0100 Subject: [PATCH 108/210] [#3184] Update Recline version The version in core was outdated and had several custom cherry-picks. This brings us in line with the latest Recline. --- ckan/public/base/css/main.css | 2 +- ckan/public/base/less/module.less | 2 +- .../reclineview/theme/public/css/recline.css | 331 +- .../reclineview/theme/public/resource.config | 25 +- .../vendor/bootstrap/2.3.2/bootstrap.js | 2280 ------ .../2.3.2/css/bootstrap-responsive.css | 1109 --- .../vendor/bootstrap/2.3.2/css/bootstrap.css | 6167 ---------------- .../2.3.2/img/glyphicons-halflings-white.png | Bin 8777 -> 0 bytes .../2.3.2/img/glyphicons-halflings.png | Bin 12799 -> 0 bytes .../bootstrap/3.2.0/css/bootstrap-theme.css | 442 ++ .../vendor/bootstrap/3.2.0/css/bootstrap.css | 6203 +++++++++++++++++ .../fonts/glyphicons-halflings-regular.eot | Bin 0 -> 20335 bytes .../fonts/glyphicons-halflings-regular.svg | 229 + .../fonts/glyphicons-halflings-regular.ttf | Bin 0 -> 41280 bytes .../fonts/glyphicons-halflings-regular.woff | Bin 0 -> 23320 bytes .../vendor/bootstrap/3.2.0/js/bootstrap.js | 2114 ++++++ .../leaflet.markercluster-src.js | 2163 ++++++ .../theme/public/vendor/recline/flot.css | 26 + .../theme/public/vendor/recline/map.css | 28 + .../theme/public/vendor/recline/recline.js | 338 +- .../theme/public/vendor/recline/slickgrid.css | 188 + .../public/vendor/slickgrid/2.0.1/README.txt | 16 - .../2.0.1/jquery-ui-1.8.16.custom.min.js | 611 -- .../slickgrid/2.0.1/jquery.event.drag-2.0.js | 6 - .../2.0.1/jquery.event.drag-2.0.min.js | 6 - .../vendor/slickgrid/2.0.1/slick.grid.js | 94 - .../vendor/slickgrid/2.0.1/slick.grid.min.css | 1 - .../vendor/slickgrid/2.0.1/slick.grid.min.js | 94 - .../slickgrid/{2.0.1 => 2.2}/MIT-LICENSE.txt | 0 .../public/vendor/slickgrid/2.2/README.md | 25 + .../2.2/controls/slick.columnpicker.css | 31 + .../2.2/controls/slick.columnpicker.js | 152 + .../slickgrid/2.2/controls/slick.pager.css | 41 + .../slickgrid/2.2/controls/slick.pager.js | 154 + .../images/ui-bg_flat_0_aaaaaa_40x100.png | Bin 0 -> 180 bytes .../images/ui-bg_flat_75_ffffff_40x100.png | Bin 0 -> 178 bytes .../images/ui-bg_glass_55_fbf9ee_1x400.png | Bin 0 -> 120 bytes .../images/ui-bg_glass_65_ffffff_1x400.png | Bin 0 -> 105 bytes .../images/ui-bg_glass_75_dadada_1x400.png | Bin 0 -> 111 bytes .../images/ui-bg_glass_75_e6e6e6_1x400.png | Bin 0 -> 110 bytes .../images/ui-bg_glass_95_fef1ec_1x400.png | Bin 0 -> 119 bytes .../ui-bg_highlight-soft_75_cccccc_1x100.png | Bin 0 -> 101 bytes .../images/ui-icons_222222_256x240.png | Bin 0 -> 4369 bytes .../images/ui-icons_2e83ff_256x240.png | Bin 0 -> 4369 bytes .../images/ui-icons_454545_256x240.png | Bin 0 -> 4369 bytes .../images/ui-icons_888888_256x240.png | Bin 0 -> 4369 bytes .../images/ui-icons_cd0a0a_256x240.png | Bin 0 -> 4369 bytes .../smoothness/jquery-ui-1.8.16.custom.css | 409 ++ .../vendor/slickgrid/2.2/images/actions.gif | Bin 0 -> 170 bytes .../2.2/images/ajax-loader-small.gif | Bin 0 -> 1849 bytes .../slickgrid/2.2/images/arrow_redo.png | Bin 0 -> 572 bytes .../2.2/images/arrow_right_peppermint.png | Bin 0 -> 128 bytes .../2.2/images/arrow_right_spearmint.png | Bin 0 -> 128 bytes .../slickgrid/2.2/images/arrow_undo.png | Bin 0 -> 578 bytes .../slickgrid/2.2/images/bullet_blue.png | Bin 0 -> 241 bytes .../slickgrid/2.2/images/bullet_star.png | Bin 0 -> 279 bytes .../2.2/images/bullet_toggle_minus.png | Bin 0 -> 154 bytes .../2.2/images/bullet_toggle_plus.png | Bin 0 -> 156 bytes .../{2.0.1 => 2.2}/images/calendar.gif | Bin .../vendor/slickgrid/2.2/images/collapse.gif | Bin 0 -> 846 bytes .../slickgrid/2.2/images/comment_yellow.gif | Bin 0 -> 257 bytes .../vendor/slickgrid/2.2/images/down.gif | Bin 0 -> 59 bytes .../slickgrid/2.2/images/drag-handle.png | Bin 0 -> 1130 bytes .../slickgrid/2.2/images/editor-helper-bg.gif | Bin 0 -> 1164 bytes .../vendor/slickgrid/2.2/images/expand.gif | Bin 0 -> 851 bytes .../vendor/slickgrid/2.2/images/header-bg.gif | Bin 0 -> 872 bytes .../2.2/images/header-columns-bg.gif | Bin 0 -> 836 bytes .../2.2/images/header-columns-over-bg.gif | Bin 0 -> 823 bytes .../vendor/slickgrid/2.2/images/help.png | Bin 0 -> 345 bytes .../vendor/slickgrid/2.2/images/info.gif | Bin 0 -> 80 bytes .../vendor/slickgrid/2.2/images/listview.gif | Bin 0 -> 2380 bytes .../vendor/slickgrid/2.2/images/pencil.gif | Bin 0 -> 914 bytes .../slickgrid/2.2/images/row-over-bg.gif | Bin 0 -> 823 bytes .../{2.0.1 => 2.2}/images/sort-asc.gif | Bin .../vendor/slickgrid/2.2/images/sort-asc.png | Bin 0 -> 105 bytes .../{2.0.1 => 2.2}/images/sort-desc.gif | Bin .../vendor/slickgrid/2.2/images/sort-desc.png | Bin 0 -> 107 bytes .../vendor/slickgrid/2.2/images/stripes.png | Bin 0 -> 1125 bytes .../vendor/slickgrid/2.2/images/tag_red.png | Bin 0 -> 537 bytes .../vendor/slickgrid/2.2/images/tick.png | Bin 0 -> 484 bytes .../slickgrid/2.2/images/user_identity.gif | Bin 0 -> 905 bytes .../2.2/images/user_identity_plus.gif | Bin 0 -> 546 bytes .../vendor/slickgrid/2.2/jquery-1.7.min.js | 4 + .../{2.0.1 => 2.2}/jquery-ui-1.8.16.custom.js | 0 .../slickgrid/2.2/jquery.event.drag-2.2.js | 402 ++ .../slickgrid/2.2/jquery.event.drop-2.2.js | 302 + .../2.2/plugins/slick.autotooltips.js | 83 + .../2.2/plugins/slick.cellcopymanager.js | 86 + .../2.2/plugins/slick.cellrangedecorator.js | 66 + .../2.2/plugins/slick.cellrangeselector.js | 113 + .../2.2/plugins/slick.cellselectionmodel.js | 154 + .../2.2/plugins/slick.checkboxselectcolumn.js | 153 + .../2.2/plugins/slick.headerbuttons.css | 39 + .../2.2/plugins/slick.headerbuttons.js | 177 + .../2.2/plugins/slick.headermenu.css | 59 + .../slickgrid/2.2/plugins/slick.headermenu.js | 275 + .../plugins/slick.rowmovemanager.js | 0 .../plugins/slick.rowselectionmodel.js | 0 .../slickgrid/2.2/slick-default-theme.css | 118 + .../public/vendor/slickgrid/2.2/slick.core.js | 467 ++ .../vendor/slickgrid/2.2/slick.dataview.js | 1126 +++ .../vendor/slickgrid/2.2/slick.editors.js | 512 ++ .../vendor/slickgrid/2.2/slick.formatters.js | 59 + .../slickgrid/{2.0.1 => 2.2}/slick.grid.css | 13 +- .../public/vendor/slickgrid/2.2/slick.grid.js | 3422 +++++++++ .../2.2/slick.groupitemmetadataprovider.js | 158 + .../vendor/slickgrid/2.2/slick.remotemodel.js | 173 + .../theme/public/vendor/timeline/LICENSE | 365 + .../theme/public/vendor/timeline/README | 1 + 109 files changed, 21054 insertions(+), 10560 deletions(-) delete mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/bootstrap.js delete mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/css/bootstrap-responsive.css delete mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/css/bootstrap.css delete mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/img/glyphicons-halflings-white.png delete mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/img/glyphicons-halflings.png create mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/css/bootstrap-theme.css create mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/css/bootstrap.css create mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.eot create mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.svg create mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.ttf create mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/fonts/glyphicons-halflings-regular.woff create mode 100644 ckanext/reclineview/theme/public/vendor/bootstrap/3.2.0/js/bootstrap.js create mode 100644 ckanext/reclineview/theme/public/vendor/leaflet.markercluster/leaflet.markercluster-src.js create mode 100644 ckanext/reclineview/theme/public/vendor/recline/flot.css create mode 100644 ckanext/reclineview/theme/public/vendor/recline/map.css create mode 100644 ckanext/reclineview/theme/public/vendor/recline/slickgrid.css delete mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.0.1/README.txt delete mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.0.1/jquery-ui-1.8.16.custom.min.js delete mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.js delete mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.0.1/jquery.event.drag-2.0.min.js delete mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.0.1/slick.grid.js delete mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.0.1/slick.grid.min.css delete mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.0.1/slick.grid.min.js rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/MIT-LICENSE.txt (100%) create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/README.md create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.columnpicker.css create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.columnpicker.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.pager.css create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/controls/slick.pager.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_flat_0_aaaaaa_40x100.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_flat_75_ffffff_40x100.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_55_fbf9ee_1x400.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_65_ffffff_1x400.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_75_dadada_1x400.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_75_e6e6e6_1x400.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_glass_95_fef1ec_1x400.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-bg_highlight-soft_75_cccccc_1x100.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_222222_256x240.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_2e83ff_256x240.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_454545_256x240.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_888888_256x240.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/images/ui-icons_cd0a0a_256x240.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/css/smoothness/jquery-ui-1.8.16.custom.css create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/actions.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/ajax-loader-small.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_redo.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_right_peppermint.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_right_spearmint.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/arrow_undo.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_blue.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_star.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_toggle_minus.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/bullet_toggle_plus.png rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/images/calendar.gif (100%) create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/collapse.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/comment_yellow.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/down.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/drag-handle.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/editor-helper-bg.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/expand.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/header-bg.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/header-columns-bg.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/header-columns-over-bg.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/help.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/info.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/listview.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/pencil.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/row-over-bg.gif rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/images/sort-asc.gif (100%) create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/sort-asc.png rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/images/sort-desc.gif (100%) create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/sort-desc.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/stripes.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/tag_red.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/tick.png create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/user_identity.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/images/user_identity_plus.gif create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery-1.7.min.js rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/jquery-ui-1.8.16.custom.js (100%) create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drag-2.2.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/jquery.event.drop-2.2.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.autotooltips.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellcopymanager.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangedecorator.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellrangeselector.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.cellselectionmodel.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.checkboxselectcolumn.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.css create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headerbuttons.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.css create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/plugins/slick.headermenu.js rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/plugins/slick.rowmovemanager.js (100%) rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/plugins/slick.rowselectionmodel.js (100%) create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick-default-theme.css create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.core.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.dataview.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.editors.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.formatters.js rename ckanext/reclineview/theme/public/vendor/slickgrid/{2.0.1 => 2.2}/slick.grid.css (95%) create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.grid.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.groupitemmetadataprovider.js create mode 100644 ckanext/reclineview/theme/public/vendor/slickgrid/2.2/slick.remotemodel.js create mode 100644 ckanext/reclineview/theme/public/vendor/timeline/LICENSE create mode 100644 ckanext/reclineview/theme/public/vendor/timeline/README diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index dfe722150a0..06ff27b7b5d 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -5763,7 +5763,7 @@ a.tag:hover { margin-top: 0; } .ckanext-datapreview > iframe { - min-height: 400px; + min-height: 650px; } .ckanext-datapreview > img { max-height: 500px; diff --git a/ckan/public/base/less/module.less b/ckan/public/base/less/module.less index 180e6815853..46a4b5c8d8e 100644 --- a/ckan/public/base/less/module.less +++ b/ckan/public/base/less/module.less @@ -170,7 +170,7 @@ margin-top: 0; & > iframe { - min-height: 400px; + min-height: 650px; } & > img { max-height: 500px; diff --git a/ckanext/reclineview/theme/public/css/recline.css b/ckanext/reclineview/theme/public/css/recline.css index 34b491742c6..722fab08867 100644 --- a/ckanext/reclineview/theme/public/css/recline.css +++ b/ckanext/reclineview/theme/public/css/recline.css @@ -20,7 +20,7 @@ body { .recline-slickgrid { height: 600px; } - +/* .recline-timeline .vmm-timeline { height: 600px; } @@ -39,11 +39,12 @@ body { margin: 0; padding: 2px 4px; } - +*/ +/* .recline-query-editor { float: right; } - +*/ .loading-spinner { float: right; background-image: url('../img/ajaxload-circle.gif'); @@ -51,6 +52,328 @@ body { height: 32px; } -.recline-pager label, .recline-query-editor label { +.recline-data-explorer .data-view-sidebar { + float: right; + margin-left: 8px; + width: 220px; +} + +.recline-data-explorer .header .navigation { + margin-bottom: 8px; +} + +.recline-data-explorer .header .navigation, +.recline-data-explorer .header .pagination, +.recline-data-explorer .header .pagination form +{ + display: inline; +} + +.recline-data-explorer .header .navigation { + float: left; +} + +.recline-data-explorer .header .menu-right { + float: right; + margin-left: 5px; + padding-left: 5px; +} + +.recline-results-info { + line-height: 35px; + margin-left: 20px; + float: left; +} + +.recline-data-explorer .data-view-sidebar > div { + margin-top: 5px; + margin-bottom: 10px; +} + +.recline-data-explorer .radio, +.recline-data-explorer .checkbox { + padding-left: 20px; +} + +.recline-data-explorer .editor-update-map { + margin: 30px 0px 20px 0px; +} + +.recline-data-explorer label { + font-weight: normal; +} + +/********************************************************** + * Query Editor + *********************************************************/ + +.recline-query-editor { + float: right; + height: 35px; + padding-right: 5px; + margin-right: 5px; + border-right: solid 2px #ddd; +} + +.header .input-prepend { + margin-bottom: auto; +} + +.header .add-on { + float: left; +} + +/* needed for Chrome but not FF */ +.header .add-on { + margin-left: -27px; +} + +/* needed for FF but not chrome */ +.header .input-prepend { + vertical-align: top; +} + +.recline-query-editor form button { + vertical-align: top; +} + +/* label for screen reader */ +.recline-query-editor .form-inline label { + position: absolute; + top:0; + left:-9999px +} + +/********************************************************** + * Pager + *********************************************************/ + +.recline-pager { + float: left; + margin: auto; + display: block; + margin-left: 20px; +} + +.recline-pager .pagination li { + display: inline-block; +} + +.recline-pager .pagination label { + display:none; +} + +.recline-pager .pagination input { + width: 40px; + height: 25px; + padding: 2px 4px; + margin: 0; + margin-top: -2px; + + border: 1px solid #cccccc; + box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + transition: border linear 0.2s, box-shadow linear 0.2s; + border-radius: 4px; + + -webkit-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -webkit-transition: border linear 0.2s, box-shadow linear 0.2s; + -webkit-border-radius: 4px; + + -moz-box-shadow: inset 0 1px 1px rgba(0, 0, 0, 0.075); + -moz-transition: border linear 0.2s, box-shadow linear 0.2s; + -moz-border-radius: 4px; + + -o-transition: border linear 0.2s, box-shadow linear 0.2s; +} + +.recline-pager .pagination a { + float: none; + margin-left: -5px; + color: #555; +} + +.recline-pager .pagination .page-range { + height: 34px; + padding: 5px 8px; + margin-left: -5px; + border: 1px solid #ddd; +} + +.recline-pager .pagination .page-range a { + padding: 0px 12px; + border: none; +} + +.recline-pager .pagination .page-range a:hover { + background-color: #ffffff; +} + +.recline-pager .pagination > li:first-child > a { + border-bottom-left-radius: 4px; + border-top-left-radius: 4px; + border-bottom-right-radius: 0px; + border-top-right-radius: 0px; + height: 34px; +} + +.recline-pager .pagination > li:last-child > a { + border-bottom-right-radius: 4px; + border-top-right-radius: 4px; + border-bottom-left-radius: 0px; + border-top-left-radius: 0px; + height: 34px; +} + +/********************************************************** + * Filter Editor + *********************************************************/ + +.recline-filter-editor { + padding: 8px; + display: none; +} + +.recline-filter-editor .filters { + margin: 20px 0px; +} + +.recline-filter-editor h3 { + margin-top: 4px; +} + +.recline-filter-editor .filter { + margin-top: 20px; +} + +.recline-filter-editor .filter .form-group { + margin-bottom: 0px; +} + +.recline-filter-editor .filter input, +.recline-filter-editor .filter label { + margin: 0px; +} + +.recline-filter-editor .js-edit button { + margin: 25px 0px 0px 0px; +} + +.recline-filter-editor .filter-term a { + font-size: 18px; +} + +.recline-filter-editor input, +.recline-filter-editor select +{ + width: 175px; +} + +.recline-filter-editor input { + margin-top: 0.5em; + margin-bottom: 10px; + -webkit-border-radius: 4px; + -moz-border-radius: 4px; + border-radius: 4px; + border: 1px solid #cccccc; +} + +.recline-filter-editor label { + font-weight: normal; + display: block; +} + +.recline-filter-editor legend { + margin-bottom: 5px; +} + +.recline-filter-editor .add-filter { + margin-top: 1em; + margin-bottom: 2em; +} + +.recline-filter-editor .update-filter { + margin-top: 1em; +} + +/********************************************************** + * Fields Widget + *********************************************************/ + +.recline-fields-view { display: none; } + +.recline-fields-view .fields-list { + padding: 0; +} + +.recline-fields-view .panel { + background-color: #f5f5f5; + border: 1px solid #e5e5e5; +} + +.recline-fields-view .panel-group h3 { + padding-left: 10px; +} + +.recline-fields-view .fields-list .panel-heading { + padding: 2px 5px; + margin: 1px 0px 1px 5px; +} + +.recline-fields-view .panel a, +.recline-fields-view .panel h4 { + display: inline; +} + +.recline-fields-view .panel a { + padding: 0; +} + +.recline-fields-view .panel h4 { + word-wrap: break-word +} + +.recline-fields-view .clear { + clear: both; +} + +.recline-fields-view .facet-items { + list-style-type: none; + margin-left: 0; +} + +.recline-fields-view .facet-item .term { + font-weight: bold; +} + +.recline-fields-view .facet-item .count { +} + +/********************************************************** + * Notifications + *********************************************************/ + +.recline-data-explorer .notification-loader { + width: 18px; + margin-left: 5px; + background-image: url(data:image/gif;base64,R0lGODlhEAAQAPQAAP///wAAAPDw8IqKiuDg4EZGRnp6egAAAFhYWCQkJKysrL6+vhQUFJycnAQEBDY2NmhoaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACH/C05FVFNDQVBFMi4wAwEAAAAh/hpDcmVhdGVkIHdpdGggYWpheGxvYWQuaW5mbwAh+QQJCgAAACwAAAAAEAAQAAAFdyAgAgIJIeWoAkRCCMdBkKtIHIngyMKsErPBYbADpkSCwhDmQCBethRB6Vj4kFCkQPG4IlWDgrNRIwnO4UKBXDufzQvDMaoSDBgFb886MiQadgNABAokfCwzBA8LCg0Egl8jAggGAA1kBIA1BAYzlyILczULC2UhACH5BAkKAAAALAAAAAAQABAAAAV2ICACAmlAZTmOREEIyUEQjLKKxPHADhEvqxlgcGgkGI1DYSVAIAWMx+lwSKkICJ0QsHi9RgKBwnVTiRQQgwF4I4UFDQQEwi6/3YSGWRRmjhEETAJfIgMFCnAKM0KDV4EEEAQLiF18TAYNXDaSe3x6mjidN1s3IQAh+QQJCgAAACwAAAAAEAAQAAAFeCAgAgLZDGU5jgRECEUiCI+yioSDwDJyLKsXoHFQxBSHAoAAFBhqtMJg8DgQBgfrEsJAEAg4YhZIEiwgKtHiMBgtpg3wbUZXGO7kOb1MUKRFMysCChAoggJCIg0GC2aNe4gqQldfL4l/Ag1AXySJgn5LcoE3QXI3IQAh+QQJCgAAACwAAAAAEAAQAAAFdiAgAgLZNGU5joQhCEjxIssqEo8bC9BRjy9Ag7GILQ4QEoE0gBAEBcOpcBA0DoxSK/e8LRIHn+i1cK0IyKdg0VAoljYIg+GgnRrwVS/8IAkICyosBIQpBAMoKy9dImxPhS+GKkFrkX+TigtLlIyKXUF+NjagNiEAIfkECQoAAAAsAAAAABAAEAAABWwgIAICaRhlOY4EIgjH8R7LKhKHGwsMvb4AAy3WODBIBBKCsYA9TjuhDNDKEVSERezQEL0WrhXucRUQGuik7bFlngzqVW9LMl9XWvLdjFaJtDFqZ1cEZUB0dUgvL3dgP4WJZn4jkomWNpSTIyEAIfkECQoAAAAsAAAAABAAEAAABX4gIAICuSxlOY6CIgiD8RrEKgqGOwxwUrMlAoSwIzAGpJpgoSDAGifDY5kopBYDlEpAQBwevxfBtRIUGi8xwWkDNBCIwmC9Vq0aiQQDQuK+VgQPDXV9hCJjBwcFYU5pLwwHXQcMKSmNLQcIAExlbH8JBwttaX0ABAcNbWVbKyEAIfkECQoAAAAsAAAAABAAEAAABXkgIAICSRBlOY7CIghN8zbEKsKoIjdFzZaEgUBHKChMJtRwcWpAWoWnifm6ESAMhO8lQK0EEAV3rFopIBCEcGwDKAqPh4HUrY4ICHH1dSoTFgcHUiZjBhAJB2AHDykpKAwHAwdzf19KkASIPl9cDgcnDkdtNwiMJCshACH5BAkKAAAALAAAAAAQABAAAAV3ICACAkkQZTmOAiosiyAoxCq+KPxCNVsSMRgBsiClWrLTSWFoIQZHl6pleBh6suxKMIhlvzbAwkBWfFWrBQTxNLq2RG2yhSUkDs2b63AYDAoJXAcFRwADeAkJDX0AQCsEfAQMDAIPBz0rCgcxky0JRWE1AmwpKyEAIfkECQoAAAAsAAAAABAAEAAABXkgIAICKZzkqJ4nQZxLqZKv4NqNLKK2/Q4Ek4lFXChsg5ypJjs1II3gEDUSRInEGYAw6B6zM4JhrDAtEosVkLUtHA7RHaHAGJQEjsODcEg0FBAFVgkQJQ1pAwcDDw8KcFtSInwJAowCCA6RIwqZAgkPNgVpWndjdyohACH5BAkKAAAALAAAAAAQABAAAAV5ICACAimc5KieLEuUKvm2xAKLqDCfC2GaO9eL0LABWTiBYmA06W6kHgvCqEJiAIJiu3gcvgUsscHUERm+kaCxyxa+zRPk0SgJEgfIvbAdIAQLCAYlCj4DBw0IBQsMCjIqBAcPAooCBg9pKgsJLwUFOhCZKyQDA3YqIQAh+QQJCgAAACwAAAAAEAAQAAAFdSAgAgIpnOSonmxbqiThCrJKEHFbo8JxDDOZYFFb+A41E4H4OhkOipXwBElYITDAckFEOBgMQ3arkMkUBdxIUGZpEb7kaQBRlASPg0FQQHAbEEMGDSVEAA1QBhAED1E0NgwFAooCDWljaQIQCE5qMHcNhCkjIQAh+QQJCgAAACwAAAAAEAAQAAAFeSAgAgIpnOSoLgxxvqgKLEcCC65KEAByKK8cSpA4DAiHQ/DkKhGKh4ZCtCyZGo6F6iYYPAqFgYy02xkSaLEMV34tELyRYNEsCQyHlvWkGCzsPgMCEAY7Cg04Uk48LAsDhRA8MVQPEF0GAgqYYwSRlycNcWskCkApIyEAOwAAAAAAAAAAAA%3D%3D); + display: inline-block; +} + +.recline-data-explorer .alert-loader { + position: absolute; + width: 200px; + left: 50%; + margin-left: -100px; + z-index: 10000; + padding: 40px 0px 40px 0px; + margin-top: -10px; + text-align: center; + font-size: 16px; + font-weight: bold; + -webkit-border-radius: 0px; + -moz-border-radius: 0px; + border-radius: 0px; + border-top: none; +} diff --git a/ckanext/reclineview/theme/public/resource.config b/ckanext/reclineview/theme/public/resource.config index b2705e40a2a..9ec647d0c51 100644 --- a/ckanext/reclineview/theme/public/resource.config +++ b/ckanext/reclineview/theme/public/resource.config @@ -17,18 +17,22 @@ main = vendor/underscore/1.4.4/underscore.js vendor/backbone/1.0.0/backbone.js vendor/mustache/0.5.0-dev/mustache.js - vendor/bootstrap/2.3.2/bootstrap.js + vendor/bootstrap/3.2.0/js/bootstrap.js vendor/json/json2.js vendor/flot/excanvas.js vendor/flot/jquery.flot.js vendor/flot/jquery.flot.time.js vendor/leaflet/0.7.3/leaflet.js vendor/leaflet.markercluster/leaflet.markercluster.js - vendor/slickgrid/2.0.1/jquery-ui-1.8.16.custom.js - vendor/slickgrid/2.0.1/jquery.event.drag-2.0.js - vendor/slickgrid/2.0.1/slick.grid.js - vendor/slickgrid/2.0.1/plugins/slick.rowselectionmodel.js - vendor/slickgrid/2.0.1/plugins/slick.rowmovemanager.js + vendor/slickgrid/2.2/jquery-ui-1.8.16.custom.js + vendor/slickgrid/2.2/jquery.event.drag-2.2.js + vendor/slickgrid/2.2/jquery.event.drop-2.2.js + vendor/slickgrid/2.2/slick.core.js + vendor/slickgrid/2.2/slick.formatters.js + vendor/slickgrid/2.2/slick.editors.js + vendor/slickgrid/2.2/slick.grid.js + vendor/slickgrid/2.2/plugins/slick.rowselectionmodel.js + vendor/slickgrid/2.2/plugins/slick.rowmovemanager.js vendor/moment/2.0.0/moment.js vendor/ckan.js/ckan.js @@ -37,11 +41,14 @@ main = widget.recordcount.js recline_view.js - vendor/bootstrap/2.3.2/css/bootstrap.css + + vendor/bootstrap/3.2.0/css/bootstrap.css vendor/leaflet/0.7.3/leaflet.css vendor/leaflet.markercluster/MarkerCluster.css vendor/leaflet.markercluster/MarkerCluster.Default.css - vendor/slickgrid/2.0.1/slick.grid.css - vendor/recline/recline.css + vendor/slickgrid/2.2/slick.grid.css + vendor/recline/slickgrid.css + vendor/recline/flot.css + vendor/recline/map.css css/recline.css diff --git a/ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/bootstrap.js b/ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/bootstrap.js deleted file mode 100644 index 44109f62d47..00000000000 --- a/ckanext/reclineview/theme/public/vendor/bootstrap/2.3.2/bootstrap.js +++ /dev/null @@ -1,2280 +0,0 @@ -/* =================================================== - * bootstrap-transition.js v2.3.2 - * http://getbootstrap.com/2.3.2/javascript.html#transitions - * =================================================== - * Copyright 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* CSS TRANSITION SUPPORT (http://www.modernizr.com/) - * ======================================================= */ - - $(function () { - - $.support.transition = (function () { - - var transitionEnd = (function () { - - var el = document.createElement('bootstrap') - , transEndEventNames = { - 'WebkitTransition' : 'webkitTransitionEnd' - , 'MozTransition' : 'transitionend' - , 'OTransition' : 'oTransitionEnd otransitionend' - , 'transition' : 'transitionend' - } - , name - - for (name in transEndEventNames){ - if (el.style[name] !== undefined) { - return transEndEventNames[name] - } - } - - }()) - - return transitionEnd && { - end: transitionEnd - } - - })() - - }) - -}(window.jQuery);/* ========================================================== - * bootstrap-alert.js v2.3.2 - * http://getbootstrap.com/2.3.2/javascript.html#alerts - * ========================================================== - * Copyright 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* ALERT CLASS DEFINITION - * ====================== */ - - var dismiss = '[data-dismiss="alert"]' - , Alert = function (el) { - $(el).on('click', dismiss, this.close) - } - - Alert.prototype.close = function (e) { - var $this = $(this) - , selector = $this.attr('data-target') - , $parent - - if (!selector) { - selector = $this.attr('href') - selector = selector && selector.replace(/.*(?=#[^\s]*$)/, '') //strip for ie7 - } - - $parent = $(selector) - - e && e.preventDefault() - - $parent.length || ($parent = $this.hasClass('alert') ? $this : $this.parent()) - - $parent.trigger(e = $.Event('close')) - - if (e.isDefaultPrevented()) return - - $parent.removeClass('in') - - function removeElement() { - $parent - .trigger('closed') - .remove() - } - - $.support.transition && $parent.hasClass('fade') ? - $parent.on($.support.transition.end, removeElement) : - removeElement() - } - - - /* ALERT PLUGIN DEFINITION - * ======================= */ - - var old = $.fn.alert - - $.fn.alert = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('alert') - if (!data) $this.data('alert', (data = new Alert(this))) - if (typeof option == 'string') data[option].call($this) - }) - } - - $.fn.alert.Constructor = Alert - - - /* ALERT NO CONFLICT - * ================= */ - - $.fn.alert.noConflict = function () { - $.fn.alert = old - return this - } - - - /* ALERT DATA-API - * ============== */ - - $(document).on('click.alert.data-api', dismiss, Alert.prototype.close) - -}(window.jQuery);/* ============================================================ - * bootstrap-button.js v2.3.2 - * http://getbootstrap.com/2.3.2/javascript.html#buttons - * ============================================================ - * Copyright 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* BUTTON PUBLIC CLASS DEFINITION - * ============================== */ - - var Button = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.button.defaults, options) - } - - Button.prototype.setState = function (state) { - var d = 'disabled' - , $el = this.$element - , data = $el.data() - , val = $el.is('input') ? 'val' : 'html' - - state = state + 'Text' - data.resetText || $el.data('resetText', $el[val]()) - - $el[val](data[state] || this.options[state]) - - // push to event loop to allow forms to submit - setTimeout(function () { - state == 'loadingText' ? - $el.addClass(d).attr(d, d) : - $el.removeClass(d).removeAttr(d) - }, 0) - } - - Button.prototype.toggle = function () { - var $parent = this.$element.closest('[data-toggle="buttons-radio"]') - - $parent && $parent - .find('.active') - .removeClass('active') - - this.$element.toggleClass('active') - } - - - /* BUTTON PLUGIN DEFINITION - * ======================== */ - - var old = $.fn.button - - $.fn.button = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('button') - , options = typeof option == 'object' && option - if (!data) $this.data('button', (data = new Button(this, options))) - if (option == 'toggle') data.toggle() - else if (option) data.setState(option) - }) - } - - $.fn.button.defaults = { - loadingText: 'loading...' - } - - $.fn.button.Constructor = Button - - - /* BUTTON NO CONFLICT - * ================== */ - - $.fn.button.noConflict = function () { - $.fn.button = old - return this - } - - - /* BUTTON DATA-API - * =============== */ - - $(document).on('click.button.data-api', '[data-toggle^=button]', function (e) { - var $btn = $(e.target) - if (!$btn.hasClass('btn')) $btn = $btn.closest('.btn') - $btn.button('toggle') - }) - -}(window.jQuery);/* ========================================================== - * bootstrap-carousel.js v2.3.2 - * http://getbootstrap.com/2.3.2/javascript.html#carousel - * ========================================================== - * Copyright 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ========================================================== */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* CAROUSEL CLASS DEFINITION - * ========================= */ - - var Carousel = function (element, options) { - this.$element = $(element) - this.$indicators = this.$element.find('.carousel-indicators') - this.options = options - this.options.pause == 'hover' && this.$element - .on('mouseenter', $.proxy(this.pause, this)) - .on('mouseleave', $.proxy(this.cycle, this)) - } - - Carousel.prototype = { - - cycle: function (e) { - if (!e) this.paused = false - if (this.interval) clearInterval(this.interval); - this.options.interval - && !this.paused - && (this.interval = setInterval($.proxy(this.next, this), this.options.interval)) - return this - } - - , getActiveIndex: function () { - this.$active = this.$element.find('.item.active') - this.$items = this.$active.parent().children() - return this.$items.index(this.$active) - } - - , to: function (pos) { - var activeIndex = this.getActiveIndex() - , that = this - - if (pos > (this.$items.length - 1) || pos < 0) return - - if (this.sliding) { - return this.$element.one('slid', function () { - that.to(pos) - }) - } - - if (activeIndex == pos) { - return this.pause().cycle() - } - - return this.slide(pos > activeIndex ? 'next' : 'prev', $(this.$items[pos])) - } - - , pause: function (e) { - if (!e) this.paused = true - if (this.$element.find('.next, .prev').length && $.support.transition.end) { - this.$element.trigger($.support.transition.end) - this.cycle(true) - } - clearInterval(this.interval) - this.interval = null - return this - } - - , next: function () { - if (this.sliding) return - return this.slide('next') - } - - , prev: function () { - if (this.sliding) return - return this.slide('prev') - } - - , slide: function (type, next) { - var $active = this.$element.find('.item.active') - , $next = next || $active[type]() - , isCycling = this.interval - , direction = type == 'next' ? 'left' : 'right' - , fallback = type == 'next' ? 'first' : 'last' - , that = this - , e - - this.sliding = true - - isCycling && this.pause() - - $next = $next.length ? $next : this.$element.find('.item')[fallback]() - - e = $.Event('slide', { - relatedTarget: $next[0] - , direction: direction - }) - - if ($next.hasClass('active')) return - - if (this.$indicators.length) { - this.$indicators.find('.active').removeClass('active') - this.$element.one('slid', function () { - var $nextIndicator = $(that.$indicators.children()[that.getActiveIndex()]) - $nextIndicator && $nextIndicator.addClass('active') - }) - } - - if ($.support.transition && this.$element.hasClass('slide')) { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $next.addClass(type) - $next[0].offsetWidth // force reflow - $active.addClass(direction) - $next.addClass(direction) - this.$element.one($.support.transition.end, function () { - $next.removeClass([type, direction].join(' ')).addClass('active') - $active.removeClass(['active', direction].join(' ')) - that.sliding = false - setTimeout(function () { that.$element.trigger('slid') }, 0) - }) - } else { - this.$element.trigger(e) - if (e.isDefaultPrevented()) return - $active.removeClass('active') - $next.addClass('active') - this.sliding = false - this.$element.trigger('slid') - } - - isCycling && this.cycle() - - return this - } - - } - - - /* CAROUSEL PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.carousel - - $.fn.carousel = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('carousel') - , options = $.extend({}, $.fn.carousel.defaults, typeof option == 'object' && option) - , action = typeof option == 'string' ? option : options.slide - if (!data) $this.data('carousel', (data = new Carousel(this, options))) - if (typeof option == 'number') data.to(option) - else if (action) data[action]() - else if (options.interval) data.pause().cycle() - }) - } - - $.fn.carousel.defaults = { - interval: 5000 - , pause: 'hover' - } - - $.fn.carousel.Constructor = Carousel - - - /* CAROUSEL NO CONFLICT - * ==================== */ - - $.fn.carousel.noConflict = function () { - $.fn.carousel = old - return this - } - - /* CAROUSEL DATA-API - * ================= */ - - $(document).on('click.carousel.data-api', '[data-slide], [data-slide-to]', function (e) { - var $this = $(this), href - , $target = $($this.attr('data-target') || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '')) //strip for ie7 - , options = $.extend({}, $target.data(), $this.data()) - , slideIndex - - $target.carousel(options) - - if (slideIndex = $this.attr('data-slide-to')) { - $target.data('carousel').pause().to(slideIndex).cycle() - } - - e.preventDefault() - }) - -}(window.jQuery);/* ============================================================= - * bootstrap-collapse.js v2.3.2 - * http://getbootstrap.com/2.3.2/javascript.html#collapse - * ============================================================= - * Copyright 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* COLLAPSE PUBLIC CLASS DEFINITION - * ================================ */ - - var Collapse = function (element, options) { - this.$element = $(element) - this.options = $.extend({}, $.fn.collapse.defaults, options) - - if (this.options.parent) { - this.$parent = $(this.options.parent) - } - - this.options.toggle && this.toggle() - } - - Collapse.prototype = { - - constructor: Collapse - - , dimension: function () { - var hasWidth = this.$element.hasClass('width') - return hasWidth ? 'width' : 'height' - } - - , show: function () { - var dimension - , scroll - , actives - , hasData - - if (this.transitioning || this.$element.hasClass('in')) return - - dimension = this.dimension() - scroll = $.camelCase(['scroll', dimension].join('-')) - actives = this.$parent && this.$parent.find('> .accordion-group > .in') - - if (actives && actives.length) { - hasData = actives.data('collapse') - if (hasData && hasData.transitioning) return - actives.collapse('hide') - hasData || actives.data('collapse', null) - } - - this.$element[dimension](0) - this.transition('addClass', $.Event('show'), 'shown') - $.support.transition && this.$element[dimension](this.$element[0][scroll]) - } - - , hide: function () { - var dimension - if (this.transitioning || !this.$element.hasClass('in')) return - dimension = this.dimension() - this.reset(this.$element[dimension]()) - this.transition('removeClass', $.Event('hide'), 'hidden') - this.$element[dimension](0) - } - - , reset: function (size) { - var dimension = this.dimension() - - this.$element - .removeClass('collapse') - [dimension](size || 'auto') - [0].offsetWidth - - this.$element[size !== null ? 'addClass' : 'removeClass']('collapse') - - return this - } - - , transition: function (method, startEvent, completeEvent) { - var that = this - , complete = function () { - if (startEvent.type == 'show') that.reset() - that.transitioning = 0 - that.$element.trigger(completeEvent) - } - - this.$element.trigger(startEvent) - - if (startEvent.isDefaultPrevented()) return - - this.transitioning = 1 - - this.$element[method]('in') - - $.support.transition && this.$element.hasClass('collapse') ? - this.$element.one($.support.transition.end, complete) : - complete() - } - - , toggle: function () { - this[this.$element.hasClass('in') ? 'hide' : 'show']() - } - - } - - - /* COLLAPSE PLUGIN DEFINITION - * ========================== */ - - var old = $.fn.collapse - - $.fn.collapse = function (option) { - return this.each(function () { - var $this = $(this) - , data = $this.data('collapse') - , options = $.extend({}, $.fn.collapse.defaults, $this.data(), typeof option == 'object' && option) - if (!data) $this.data('collapse', (data = new Collapse(this, options))) - if (typeof option == 'string') data[option]() - }) - } - - $.fn.collapse.defaults = { - toggle: true - } - - $.fn.collapse.Constructor = Collapse - - - /* COLLAPSE NO CONFLICT - * ==================== */ - - $.fn.collapse.noConflict = function () { - $.fn.collapse = old - return this - } - - - /* COLLAPSE DATA-API - * ================= */ - - $(document).on('click.collapse.data-api', '[data-toggle=collapse]', function (e) { - var $this = $(this), href - , target = $this.attr('data-target') - || e.preventDefault() - || (href = $this.attr('href')) && href.replace(/.*(?=#[^\s]+$)/, '') //strip for ie7 - , option = $(target).data('collapse') ? 'toggle' : $this.data() - $this[$(target).hasClass('in') ? 'addClass' : 'removeClass']('collapsed') - $(target).collapse(option) - }) - -}(window.jQuery);/* ============================================================ - * bootstrap-dropdown.js v2.3.2 - * http://getbootstrap.com/2.3.2/javascript.html#dropdowns - * ============================================================ - * Copyright 2013 Twitter, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ============================================================ */ - - -!function ($) { - - "use strict"; // jshint ;_; - - - /* DROPDOWN CLASS DEFINITION - * ========================= */ - - var toggle = '[data-toggle=dropdown]' - , Dropdown = function (element) { - var $el = $(element).on('click.dropdown.data-api', this.toggle) - $('html').on('click.dropdown.data-api', function () { - $el.parent().removeClass('open') - }) - } - - Dropdown.prototype = { - - constructor: Dropdown - - , toggle: function (e) { - var $this = $(this) - , $parent - , isActive - - if ($this.is('.disabled, :disabled')) return - - $parent = getParent($this) - - isActive = $parent.hasClass('open') - - clearMenus() - - if (!isActive) { - if ('ontouchstart' in document.documentElement) { - // if mobile we we use a backdrop because click events don't delegate - $('