From 845c0e5769edb6a1e66a86d97e3ae67f10841363 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 31 May 2012 11:13:14 +0100 Subject: [PATCH 01/11] [#2389] Move recline js and css to a snippet This will make easier to modify the libraries and load them conditionally. --- ckan/templates/package/resource_read.html | 36 ++---------------- .../snippets/recline-extra-footer.html | 13 +++++++ .../snippets/recline-extra-header.html | 38 +++++++++++++++++++ 3 files changed, 54 insertions(+), 33 deletions(-) create mode 100644 ckan/templates/snippets/recline-extra-footer.html create mode 100644 ckan/templates/snippets/recline-extra-header.html diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index bebe246a0ae..cf03b67f502 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -15,36 +15,9 @@ - - - - - - - - - + + From d8734c0064d0f1264408fc20d332c42b7ab09019 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 10:24:48 +0100 Subject: [PATCH 02/11] [#2389] Remove Datastore option from the resource form The datastore checkbox was confusing so we simplified the process. All resources have the datastore potentially enabled, and only after an actual write, the controller sets the 'webstore_url' extra to active. --- ckan/public/scripts/application.js | 19 ++----------------- ckan/public/scripts/templates.js | 11 ----------- ckan/templates/js_strings.html | 1 - 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 72124f00f18..11202ad634d 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -473,7 +473,7 @@ CKAN.View.ResourceEditor = Backbone.View.extend({ CKAN.View.Resource = Backbone.View.extend({ initialize: function() { this.el = $(this.el); - _.bindAll(this,'updateName','updateIcon','name','askToDelete','openMyPanel','setErrors','setupDynamicExtras','addDynamicExtra', 'onDatastoreEnabledChange'); + _.bindAll(this,'updateName','updateIcon','name','askToDelete','openMyPanel','setErrors','setupDynamicExtras','addDynamicExtra' ); this.render(); }, render: function() { @@ -508,12 +508,8 @@ CKAN.View.Resource = Backbone.View.extend({ // Hook to open panel link this.li.find('.resource-open-my-panel').click(this.openMyPanel); this.table.find('.js-resource-edit-delete').click(this.askToDelete); - this.table.find('.js-datastore-enabled-checkbox').change(this.onDatastoreEnabledChange); // Hook to markdown editor CKAN.Utils.setupMarkdownEditor(this.table.find('.markdown-editor')); - if (resource_object.resource.webstore_url) { - this.table.find('.js-datastore-enabled-checkbox').prop('checked', true); - } // Set initial state this.updateName(); @@ -711,12 +707,6 @@ CKAN.View.Resource = Backbone.View.extend({ removeFromDom: function() { this.li.remove(); this.table.remove(); - }, - onDatastoreEnabledChange: function(e) { - var isChecked = this.table.find('.js-datastore-enabled-checkbox').prop('checked'); - var webstore_url = isChecked ? 'enabled' : null; - this.model.set({webstore_url: webstore_url}); - this.table.find('.js-datastore-enabled-text').val(webstore_url); } }); @@ -849,7 +839,6 @@ CKAN.View.ResourceAddUpload = Backbone.View.extend({ , hash: data._checksum , cache_url: data._location , cache_url_updated: lastmod - , webstore_url: data._location } , { error: function(model, error) { @@ -916,7 +905,6 @@ CKAN.View.ResourceAddUrl = Backbone.View.extend({ size: data.size, mimetype: data.mimetype, last_modified: data.last_modified, - webstore_url: 'enabled', url_error: (data.url_errors || [""])[0] }); self.collection.add(newResource); @@ -926,9 +914,6 @@ CKAN.View.ResourceAddUrl = Backbone.View.extend({ } else { newResource.set({url: urlVal, resource_type: this.options.mode}); - if (newResource.get('resource_type')=='file') { - newResource.set({webstore_url: 'enabled'}); - } this.collection.add(newResource); this.resetForm(); } @@ -1657,7 +1642,7 @@ CKAN.DataPreview = function ($, my) { } // 4 situations - // a) have a webstore_url + // a) webstore_url is active (something was posted to the datastore) // b) csv or xls (but not webstore) // c) can be treated as plain text // d) none of the above but worth iframing (assumption is diff --git a/ckan/public/scripts/templates.js b/ckan/public/scripts/templates.js index 6ecad116a6b..596c94f45fc 100644 --- a/ckan/public/scripts/templates.js +++ b/ckan/public/scripts/templates.js @@ -27,7 +27,6 @@ CKAN.Templates.resourceEntry = ' \ '; var youCanUseMarkdownString = CKAN.Strings.youCanUseMarkdown.replace('%a', '').replace('%b', ''); -var shouldADataStoreBeEnabledString = CKAN.Strings.shouldADataStoreBeEnabled.replace('%a', '').replace('%b', ''); var datesAreInISOString = CKAN.Strings.datesAreInISO.replace('%a', '').replace('%b', '').replace('%c', '').replace('%d', ''); // TODO it would be nice to unify this with the markdown editor specified in helpers.py @@ -93,16 +92,6 @@ CKAN.Templates.resourceDetails = ' \ {{/if}} \ \ \ -
\ - \ -
\ - \ -
\ -
\
\ \
\ diff --git a/ckan/templates/js_strings.html b/ckan/templates/js_strings.html index 1b757bb1115..61db8cf8e29 100644 --- a/ckan/templates/js_strings.html +++ b/ckan/templates/js_strings.html @@ -65,7 +65,6 @@ addExtraField = _('Add Extra Field'), deleteResource = _('Delete Resource'), youCanUseMarkdown = _('You can use %aMarkdown formatting%b here.'), - shouldADataStoreBeEnabled = _('Should a %aDataStore table and Data API%b be enabled for this resource?'), datesAreInISO = _('Dates are in %aISO Format%b — eg. %c2012-12-25%d or %c2010-05-31T14:30%d.'), dataFileUploaded = _('Data File (Uploaded)') ), indent=4)} From 8ebf3118f717f7b7deaeedbe6644b311f43d1a23 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 10:28:31 +0100 Subject: [PATCH 03/11] [#2389,#2472] Remove datastore enabled checks from controller Datastore is now enabled by default for all resources. When something is posted to the datastore, the 'webstore_url' extra is set to 'active'. Also replaced model calls with logic functions. --- ckan/controllers/datastore.py | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/ckan/controllers/datastore.py b/ckan/controllers/datastore.py index bd5e2d124e2..5a8d5526c32 100644 --- a/ckan/controllers/datastore.py +++ b/ckan/controllers/datastore.py @@ -1,9 +1,9 @@ from ckan.lib.base import BaseController, abort, _, c, response, request, g import ckan.model as model -from ckan.lib.helpers import json from ckan.lib.jsonp import jsonpify from ckan.logic import get_action, check_access -from ckan.logic import NotFound, NotAuthorized, ValidationError +from ckan.logic import NotFound, NotAuthorized + class DatastoreController(BaseController): def _make_redirect(self, id, url=''): @@ -20,10 +20,6 @@ def read(self, id, url=''): try: resource = get_action('resource_show')(context, {'id': id}) - if not resource.get('webstore_url', ''): - return { - 'error': 'DataStore is disabled for this resource' - } self._make_redirect(id, url) return '' except NotFound: @@ -36,19 +32,15 @@ def write(self, id, url): context = {'model': model, 'session': model.Session, 'user': c.user or c.author} try: - resource = model.Resource.get(id) - if not resource: - abort(404, _('Resource not found')) - if not resource.webstore_url: - return { - 'error': 'DataStore is disabled for this resource' - } - context["resource"] = resource check_access('resource_update', context, {'id': id}) + resource_dict = get_action('resource_show')(context,{'id':id}) + if not resource_dict['webstore_url']: + resource_dict['webstore_url'] = u'active' + get_action('resource_update')(context,resource_dict) + self._make_redirect(id, url) return '' except NotFound: abort(404, _('Resource not found')) except NotAuthorized: abort(401, _('Unauthorized to read resource %s') % id) - From 33f42937f1f1cb2c524dcc20e0f32734f6a05bc2 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 10:38:07 +0100 Subject: [PATCH 04/11] [#2604] Add requests as requirement for tests --- pip-requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/pip-requirements-test.txt b/pip-requirements-test.txt index 41a359c0264..b8103e124eb 100644 --- a/pip-requirements-test.txt +++ b/pip-requirements-test.txt @@ -1,5 +1,6 @@ # These are packages that required when running ckan tests nose +requests==0.6.4 -e git+https://github.com/okfn/ckanclient#egg=ckanclient From 16320b20476295c20f7602b542d4f3b085c698dc Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 10:47:09 +0100 Subject: [PATCH 05/11] [#2389] New tests for the datastore A mock class mimics Nginx forwarding requests to Elastic Search, so we can actually test what is being returned. --- ckan/tests/functional/test_datastore.py | 255 +++++++++++++++++------- 1 file changed, 183 insertions(+), 72 deletions(-) diff --git a/ckan/tests/functional/test_datastore.py b/ckan/tests/functional/test_datastore.py index 302bccdac0d..27a85e0211c 100644 --- a/ckan/tests/functional/test_datastore.py +++ b/ckan/tests/functional/test_datastore.py @@ -1,92 +1,203 @@ +import socket import json -from nose.tools import assert_equal +import time +import requests + +from nose.plugins.skip import SkipTest from pylons import config -from ckan.tests import * -from ckan.tests.pylons_controller import PylonsTestCase +from ckan.tests import TestController, CreateTestData import ckan.model as model +from ckan.lib.helpers import url_for + + +ELASTIC_SEARCH_HOST = config.get('elastic_search_host', '0.0.0.0: 9200') + + +class MockResponse(object): + '''Mock Requests response + + A mock object to make paster responses look like the ones returned + by Requests (only the useful properties are used) + + ''' + + def __init__(self, res): + ''' + : param res: a response object returned by the paster test app + ''' + self.status_code = res.status + self.content = res.body + self.headers = res.header_dict + + +class MockProxyServer(object): + '''Mock proxy server to Elastic Search + + A mock class used to mimic Nginx forwarding to Elastic Search when + querying the CKAN data API. It basically does two requests, one to + the CKAN controller, and a second one to Elastic Search, using the + value of the 'X-Accel-Redirect' header. + + ''' + + def __init__(self, app): + ''' + : param app: a paster test application + ''' + + self.elastic_search_host = ELASTIC_SEARCH_HOST + + self.app = app + + def _get_elastic_search_offset(self, res): + ''' + : param res: a response object returned by the paster test app + ''' + + redirect = dict(res.headers).get('X-Accel-Redirect') + + # Remove the /elastic bit + return redirect[8:] if redirect else None + + def _forward_request(self, res, data=None): + ''' + : param res: a response object returned by the paster test app + : param data: a dictionary of data to be sent to ES. Will produce + a POST request + ''' + + elastic_search_offset = self._get_elastic_search_offset(res) + + if res.status != 200 and not elastic_search_offset: + # No need to forward, return a Requests-like response + return MockResponse(res) + + assert elastic_search_offset + + if data: + res = requests.post('http: //%s%s' % (self.elastic_search_host, elastic_search_offset), data=json.dumps(data)) + else: + res = requests.get('http: //%s%s' % (self.elastic_search_host, elastic_search_offset)) + + return res + + def get(self, offset, ckan_status=200): + ''' + : param offset: CKAN route to request + : param ckan_status: expected status to be returned, will throw an + exception if different from the actually returned + ''' + + res = self.app.get(offset, status=ckan_status) + return self._forward_request(res) + + def post(self, offset, data=None, ckan_status=200): + ''' + : param offset: CKAN route to request + : param data: a dictionary of data to be sent to ES. + : param ckan_status: expected status to be returned, will throw an + exception if different from the actually returned + ''' + + res = self.app.post(offset, params=data, status=ckan_status) + return self._forward_request(res, data) + + +class TestDatastoreController(TestController): -class TestWebstoreController(TestController, PylonsTestCase): @classmethod def setup_class(cls): - PylonsTestCase.setup_class() + + # Check if Elastic Search is available, otherwise skip tests + try: + requests.get('http: //%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) + except (requests.exceptions.RequestException, socket.timeout), e: + raise SkipTest('Could not reach Elastic Search, skipping... (%r)' % e) + + # Push dummy data to create the test index + requests.put('http: //%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) + model.repo.init_db() CreateTestData.create() - + @classmethod def teardown_class(self): + + # Delete the test index on ES + requests.delete('http: //%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) + model.repo.rebuild_db() # TODO: do we test authz. In essence authz is same as for resource read / # edit which in turn is same as dataset read / edit and which is tested # extensively elsewhere ... - def test_read(self): - dataset = model.Package.by_name(CreateTestData.pkg_names[0]) - resource_id = dataset.resources[0].id - offset = url_for('datastore_read', id=resource_id) - res = self.app.get(offset) - assert_equal(res.status, 200) - data = json.loads(res.body) - assert 'error' in data - - dataset.resources[0].webstore_url = 'enabled' - model.repo.new_revision() - model.Session.add(dataset.resources[0]) - model.Session.commit() - - offset = url_for('datastore_read', id=resource_id) - res = self.app.get(offset) - assert_equal(res.status, 200) - assert_equal(res.body, '""') - headers = dict(res.headers) - assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s?' - % (config['ckan.site_id'], resource_id)) - - offset = url_for('datastore_read', id=resource_id, url='/_search') - res = self.app.get(offset) - assert_equal(res.status, 200) - headers = dict(res.headers) - assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s/_search?' - % (config['ckan.site_id'], resource_id)) - - def test_update(self): + def test_basic(self): + + mock_server = MockProxyServer(self.app) + + # Create a dataset, the resource should have the webstore_url + # extra empty dataset = model.Package.by_name(CreateTestData.pkg_names[0]) resource_id = dataset.resources[0].id - offset = url_for('datastore_write', id='does-not-exist') - res = self.app.post(offset, status=404) - assert res.status == 404 - - dataset.resources[0].webstore_url = '' - model.repo.new_revision() - model.Session.add(dataset.resources[0]) - model.Session.commit() - - offset = url_for('datastore_write', id=resource_id) - res = self.app.post(offset) - assert res.status == 200 - print res.body - assert 'error' in json.loads(res.body) - - dataset.resources[0].webstore_url = 'enabled' - model.repo.new_revision() - model.Session.add(dataset.resources[0]) - model.Session.commit() - - offset = url_for('datastore_write', id=resource_id) - res = self.app.post(offset) - # in fact visitor can edit! - # assert res.status in [401,302], res.status - assert res.status == 200 - headers = dict(res.headers) - assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s?' - % (config['ckan.site_id'], resource_id)) - - - offset = url_for('datastore_write', id=resource_id, url='/_mapping') - res = self.app.post(offset) - assert res.status == 200 - headers = dict(res.headers) - assert_equal(headers['X-Accel-Redirect'], '/elastic/ckan-%s/%s/_mapping?' - % (config['ckan.site_id'], resource_id)) + assert not dataset.resources[0].webstore_url + offset_wrong_read = url_for('datastore_read', id='wrong_resource_id') + offset_wrong_write = url_for('datastore_write', id='wrong_resource_id') + offset_read = url_for('datastore_read', id=resource_id) + offset_write = url_for('datastore_write', id=resource_id) + + # Resource not found + res = mock_server.get(offset_wrong_read, ckan_status=404) + assert 'Resource not found' in res.content + res = mock_server.post(offset_wrong_write, ckan_status=404, data={'a': 1}) + assert 'Resource not found' in res.content + + # Empty datastore + res = mock_server.get(offset_read) + + assert res.status_code == 400 + assert 'No handler found for uri' in res.content + assert resource_id in res.content + + res = mock_server.get(offset_read + '/_mapping') + content = json.loads(res.content) + + assert res.status_code == 404 + assert content['status'] == 404 + assert 'TypeMissingException' in content['error'] + assert resource_id in content['error'] + + # Push some stuff via the data API + data = {'a': 1, 'b': 2.78, 'c': 'test'} + res = mock_server.post(offset_write + '/1', data) + + content = json.loads(res.content) + assert content['ok'] == True + assert content['_index'] == 'ckan-%s' % config.get('ckan.site_id') + + # We need to wait or ES returns 0 hits + time.sleep(1) + + # Query the data API + res = mock_server.get(offset_read + '/_mapping') + content = json.loads(res.content) + + assert content == {resource_id: { + 'properties': { + 'a': {'type': 'long'}, + 'b': {'type': 'double'}, + 'c': {'type': 'string'} + } + } + } + + res = mock_server.get(offset_read + '/_search') + content = json.loads(res.content) + assert content['hits']['total'] == 1 + assert content['hits']['hits'][0]['_source'] == data + + # Check that the webstore_url extra was set to active + dataset = model.Package.by_name(CreateTestData.pkg_names[0]) + assert dataset.resources[0].webstore_url == u'active' From bd817df3cda3b0fc2e7a20d0b4808c022215cf50 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 17:53:55 +0100 Subject: [PATCH 06/11] [#2389] Try to make the data explorer fail with more dignity Adds some checks for DataStore and DataProxy backends. Recline does not make easy to catch exceptions and errors while initializing, but hopefully this will cover the most common cases. Note: includes a small patch on recline.js, which will be ported upstream. --- ckan/public/scripts/application.js | 39 +++++++++++++++++-- ckan/public/scripts/vendor/recline/recline.js | 5 +++ ckan/templates/js_strings.html | 6 ++- 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 11202ad634d..0eb88162420 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -1553,6 +1553,14 @@ CKAN.DataPreview = function ($, my) { my.loadPreviewDialog = function(resourceData) { my.$dialog.html('

Loading ...

'); + function showError(msg){ + msg = msg || CKAN.Strings.errorLoadingPreview; + return $('#ckanext-datapreview') + .append('
') + .addClass('alert alert-error fade in') + .html(msg); + } + function initializeDataExplorer(dataset) { var views = [ { @@ -1586,6 +1594,7 @@ CKAN.DataPreview = function ($, my) { } }); + // ----------------------------- // Setup the Embed modal dialog. // ----------------------------- @@ -1665,14 +1674,38 @@ CKAN.DataPreview = function ($, my) { if (resourceData.webstore_url) { resourceData.elasticsearch_url = '/api/data/' + resourceData.id; var dataset = new recline.Model.Dataset(resourceData, 'elasticsearch'); - initializeDataExplorer(dataset); + var errorMsg = CKAN.Strings.errorLoadingPreview + ': ' + CKAN.Strings.errorDataStore; + dataset.fetch() + .done(function(dataset){ + initializeDataExplorer(dataset); + }) + .fail(function(error){ + if (error.message) errorMsg += ' (' + error.message + ')'; + showError(errorMsg); + }); + } else if (resourceData.formatNormalized in {'csv': '', 'xls': ''}) { // set format as this is used by Recline in setting format for DataProxy resourceData.format = resourceData.formatNormalized; var dataset = new recline.Model.Dataset(resourceData, 'dataproxy'); - initializeDataExplorer(dataset); - $('.recline-query-editor .text-query').hide(); + var errorMsg = CKAN.Strings.errorLoadingPreview + ': ' +CKAN.Strings.errorDataProxy; + dataset.fetch() + .done(function(dataset){ + + dataset.bind('query:fail', function(error) { + $('#ckanext-datapreview .data-view-container').hide(); + $('#ckanext-datapreview .header').hide(); + $('.preview-header .btn').hide(); + }); + + initializeDataExplorer(dataset); + $('.recline-query-editor .text-query').hide(); + }) + .fail(function(error){ + if (error.message) errorMsg += ' (' + error.message + ')'; + showError(errorMsg); + }); } else if (resourceData.formatNormalized in { 'rdf+xml': '', diff --git a/ckan/public/scripts/vendor/recline/recline.js b/ckan/public/scripts/vendor/recline/recline.js index 271e9c54fb9..39ca7c9c737 100644 --- a/ckan/public/scripts/vendor/recline/recline.js +++ b/ckan/public/scripts/vendor/recline/recline.js @@ -3177,6 +3177,11 @@ this.recline.Backend = this.recline.Backend || {}; var dfd = $.Deferred(); this._wrapInTimeout(jqxhr).done(function(schema) { // only one top level key in ES = the type so we can ignore it + // CKAN + if (!schema){ + dfd.reject({'message':'Elastic Search did not return a mapping'}); + return; + } var key = _.keys(schema)[0]; var fieldData = _.map(schema[key].properties, function(dict, fieldName) { dict.id = fieldName; diff --git a/ckan/templates/js_strings.html b/ckan/templates/js_strings.html index 61db8cf8e29..77314fb993f 100644 --- a/ckan/templates/js_strings.html +++ b/ckan/templates/js_strings.html @@ -66,7 +66,11 @@ deleteResource = _('Delete Resource'), youCanUseMarkdown = _('You can use %aMarkdown formatting%b here.'), datesAreInISO = _('Dates are in %aISO Format%b — eg. %c2012-12-25%d or %c2010-05-31T14:30%d.'), - dataFileUploaded = _('Data File (Uploaded)') + dataFileUploaded = _('Data File (Uploaded)'), + errorLoadingPreview = _('Could not load preview'), + errorDataProxy = _('DataProxy returned an error'), + errorDataStore = _('DataStore returned an error') + ), indent=4)} From 7f59a58e15dfc1d2498534ef0d474cc7ea08d2b9 Mon Sep 17 00:00:00 2001 From: amercader Date: Wed, 27 Jun 2012 18:13:57 +0100 Subject: [PATCH 07/11] Update datastore docs --- doc/datastore.rst | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/doc/datastore.rst b/doc/datastore.rst index 15fcd8fe35c..9f3de6a919c 100644 --- a/doc/datastore.rst +++ b/doc/datastore.rst @@ -66,19 +66,41 @@ Please see the ElasticSearch_ documentation. 2. Configure Nginx ------------------ -You must add to your Nginx CKAN site entry the following:: - - location /elastic/ { - internal; - # location of elastic search - proxy_pass http://0.0.0.0:9200/; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +As previously mentioned, Nginx will be used on top of CKAN to forward +requests to Elastic Search. CKAN will still be served by Apache or the +development server (Paster), but all requests will be forwarded to it +by Ngnix. + +This is an example of an Nginx configuration file. Note the two locations +defined, `/` will point to the server running CKAN (Apache or Paster), and +`/elastic/` to the Elastic Search instance:: + + server { + listen 80 default; + server_name localhost; + + access_log /var/log/nginx/localhost.access.log; + + location / { + # location of apache or ckan under paster + proxy_pass http://127.0.0.1:8080; + proxy_set_header Host $host; + } + location /elastic/ { + internal; + # location of elastic search + proxy_pass http://:127.0.0.1:9200/; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } } .. note:: update the proxy_pass field value to point to your ElasticSearch instance (if it is not localhost and default port). +Remember that after setting up Nginx, you need to access CKAN via its port +(80), not the Apache or Paster (5000) one, otherwise the DataStore won't work. + 3. Enable datastore features in CKAN ------------------------------------ From 5e165b9e38f716b38907e3ac3e162f62c221d0d9 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Jun 2012 10:51:29 +0100 Subject: [PATCH 08/11] [#2389] Fix silly errors on datastore tests --- ckan/public/scripts/application.js | 4 ++-- ckan/tests/functional/test_datastore.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 0eb88162420..d49af47cf4a 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -1001,7 +1001,7 @@ CKAN.Utils = function($, my) { input_box.attr('name', new_name); input_box.attr('id', new_name); - + var $new = $('

'); $new.append($('').attr('name', old_name).val(ui.item.value)); $new.append(' '); @@ -1447,7 +1447,7 @@ CKAN.Utils = function($, my) { }, }); }; - + // This only needs to happen on dataset pages, but it doesn't seem to do // any harm to call it anyway. $('#user_follow_button').on('click', followButtonClicked); diff --git a/ckan/tests/functional/test_datastore.py b/ckan/tests/functional/test_datastore.py index 27a85e0211c..2fb2d005460 100644 --- a/ckan/tests/functional/test_datastore.py +++ b/ckan/tests/functional/test_datastore.py @@ -76,9 +76,9 @@ def _forward_request(self, res, data=None): assert elastic_search_offset if data: - res = requests.post('http: //%s%s' % (self.elastic_search_host, elastic_search_offset), data=json.dumps(data)) + res = requests.post('http://%s%s' % (self.elastic_search_host, elastic_search_offset), data=json.dumps(data)) else: - res = requests.get('http: //%s%s' % (self.elastic_search_host, elastic_search_offset)) + res = requests.get('http://%s%s' % (self.elastic_search_host, elastic_search_offset)) return res @@ -111,12 +111,12 @@ def setup_class(cls): # Check if Elastic Search is available, otherwise skip tests try: - requests.get('http: //%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) + requests.get('http://%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) except (requests.exceptions.RequestException, socket.timeout), e: raise SkipTest('Could not reach Elastic Search, skipping... (%r)' % e) # Push dummy data to create the test index - requests.put('http: //%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) + requests.put('http://%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) model.repo.init_db() CreateTestData.create() @@ -125,7 +125,7 @@ def setup_class(cls): def teardown_class(self): # Delete the test index on ES - requests.delete('http: //%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) + requests.delete('http://%s/ckan-%s/' % (ELASTIC_SEARCH_HOST, config.get('ckan.site_id'))) model.repo.rebuild_db() From c8bc38c62e86a6c834c82fbad78ab233515ead43 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 28 Jun 2012 11:10:55 +0100 Subject: [PATCH 09/11] [#2389] Fix tests that were failing if 'error' was present on the JS strings --- ckan/tests/functional/test_group.py | 2 +- ckan/tests/functional/test_package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index e71cd286b1e..8b89aa6c7d9 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -460,7 +460,7 @@ def test_1_do_diff(self): res = form.submit() res = res.follow() main_res = self.main_div(res) - assert 'error' not in main_res.lower(), main_res + assert 'form-errors' not in main_res.lower(), main_res assert 'Revision Differences' in main_res, main_res assert self.grp.name in main_res, main_res assert 'description
- Written by Puccini\n+ Written off
' in main_res, main_res diff --git a/ckan/tests/functional/test_package.py b/ckan/tests/functional/test_package.py index 33d88f4f22c..ffe423737fb 100644 --- a/ckan/tests/functional/test_package.py +++ b/ckan/tests/functional/test_package.py @@ -1446,7 +1446,7 @@ def test_1_do_diff(self): res = form.submit() res = res.follow() main_res = self.main_div(res) - assert 'error' not in main_res.lower(), main_res + assert 'form-errors' not in main_res.lower(), main_res assert 'Revision Differences' in main_res, main_res assert self.pkg1.name in main_res, main_res assert 'notes
- Written by Puccini\n+ Written off
' in main_res, main_res From 5fa209a41e2b01faf629ce9fa91bb8c7eb172ed5 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 29 Jun 2012 17:49:25 +0100 Subject: [PATCH 10/11] [#2389] Be more strict when checking for ES --- ckan/tests/functional/test_datastore.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ckan/tests/functional/test_datastore.py b/ckan/tests/functional/test_datastore.py index 2fb2d005460..1827b2adf70 100644 --- a/ckan/tests/functional/test_datastore.py +++ b/ckan/tests/functional/test_datastore.py @@ -111,7 +111,17 @@ def setup_class(cls): # Check if Elastic Search is available, otherwise skip tests try: - requests.get('http://%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) + res = requests.get('http://%s/_status' % ELASTIC_SEARCH_HOST, timeout=2) + if res.status_code == 200: + try: + content = json.loads(res.content) + if not 'ok'in content or not '_shards' in content: + raise ValueError + except ValueError: + raise SkipTest('This does not look like Elastic Search, skipping...') + else: + raise SkipTest('Could not reach Elastic Search, skipping...') + except (requests.exceptions.RequestException, socket.timeout), e: raise SkipTest('Could not reach Elastic Search, skipping... (%r)' % e) From 71278f801b6bf734c7a9a5ddb745c556c0c3d866 Mon Sep 17 00:00:00 2001 From: amercader Date: Fri, 29 Jun 2012 17:51:57 +0100 Subject: [PATCH 11/11] [#2389] Yet more failing errors if 'error' was present on the JS strings --- ckan/tests/functional/test_authz.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ckan/tests/functional/test_authz.py b/ckan/tests/functional/test_authz.py index c70fa28982f..6b2e55dd62b 100644 --- a/ckan/tests/functional/test_authz.py +++ b/ckan/tests/functional/test_authz.py @@ -115,7 +115,9 @@ def _test_via_wui(self, action, user, entity_name, entity='dataset'): r = res.body r = r.replace('form_errors', '') - tests['error string'] = bool('error' not in r) + # Commenting as it seems a very ineffective way of checking for errors + # (e.g. tests fail if there is a JS string with the word 'error' on it) + # tests['error string'] = bool('error' not in r) tests['status'] = bool(res.status in (200, 201)) tests['0 packages found'] = bool(u'0 packages found' not in res) is_ok = False not in tests.values()