From fca35e5f6929e3008640be1d1c2460dbc2a2ba32 Mon Sep 17 00:00:00 2001 From: Toby Date: Thu, 28 Jun 2012 13:48:56 +0100 Subject: [PATCH 001/103] [#1387] fix resource urls with no protocols --- ckan/lib/dictization/model_dictize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 74e15ed8baa..d3a3620fc6b 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -102,6 +102,10 @@ def resource_dictize(res, context): model = context['model'] tracking = model.TrackingSummary.get_for_resource(res.url) resource['tracking_summary'] = tracking + # some urls do not have the protocol this adds http:// to these + url = resource['url'] + if not (url.startswith('http://') or url.startswith('https://')): + resource['url'] = u'http://' + url return resource def related_dictize(rel, context): From 85d88059e921deb93025d9d8ef55dc551f1f788d Mon Sep 17 00:00:00 2001 From: Toby Date: Thu, 28 Jun 2012 14:31:46 +0100 Subject: [PATCH 002/103] [#1387] fix tests with bad urls --- ckan/tests/lib/test_dictization.py | 16 ++++++++-------- ckan/tests/logic/test_action.py | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index 7b986d6c120..aadfcbebe18 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -364,7 +364,7 @@ def test_09_package_alter(self): anna_dictized = package_dictize(anna1, context) anna_dictized["name"] = u'annakarenina_changed' - anna_dictized["resources"][0]["url"] = u'new_url' + anna_dictized["resources"][0]["url"] = u'http://new_url' model.repo.new_revision() package_dict_save(anna_dictized, context) @@ -402,7 +402,7 @@ def test_10_package_alter_pending(self): anna_dictized = package_dictize(anna1, context) anna_dictized['name'] = u'annakarenina_changed2' - anna_dictized['resources'][0]['url'] = u'new_url2' + anna_dictized['resources'][0]['url'] = u'http://new_url2' anna_dictized['tags'][0]['name'] = u'new_tag' anna_dictized['tags'][0].pop('id') #test if anna_dictized['extras'][0]['value'] = u'"new_value"' @@ -496,7 +496,7 @@ def test_11_add_pending(self): anna_dictized['notes'] = 'wee' anna_dictized['resources'].append({ 'format': u'plain text', - 'url': u'newurl'} + 'url': u'http://newurl'} ) anna_dictized['tags'].append({'name': u'newnew_tag'}) anna_dictized['extras'].append({'key': 'david', @@ -676,7 +676,7 @@ def test_13_get_package_in_past(self): second_dictized = self.remove_changable_columns(package_dictize(anna1, context)) first_dictized["name"] = u'annakarenina_changed' - first_dictized["resources"][0]["url"] = u'new_url' + first_dictized["resources"][0]["url"] = u'http://new_url' assert second_dictized == first_dictized @@ -684,7 +684,7 @@ def test_13_get_package_in_past(self): third_dictized = self.remove_changable_columns(package_dictize(anna1, context)) second_dictized['name'] = u'annakarenina_changed2' - second_dictized['resources'][0]['url'] = u'new_url2' + second_dictized['resources'][0]['url'] = u'http://new_url2' second_dictized['tags'][0]['name'] = u'new_tag' second_dictized['tags'][0]['display_name'] = u'new_tag' second_dictized['extras'][0]['value'] = u'"new_value"' @@ -711,7 +711,7 @@ def test_13_get_package_in_past(self): u'size': None, u'state': u'active', u'tracking_summary': {'total': 0, 'recent': 0}, - u'url': u'newurl', + u'url': u'http://newurl', u'webstore_last_updated': None, u'webstore_url': None}) @@ -741,7 +741,7 @@ def test_14_resource_no_id(self): 'description': u'Full text. Needs escaping: " Umlaut: \xfc', 'format': u'plain text', 'tracking_summary': {'recent': 0, 'total': 0}, - 'url': u'test_new', + 'url': u'http://test_new', 'cache_url': None, 'webstore_url': None, 'cache_last_updated': None, @@ -760,7 +760,7 @@ def test_14_resource_no_id(self): model.Session.commit() model.Session.remove() - res = model.Session.query(model.Resource).filter_by(url=u'test_new').one() + res = model.Session.query(model.Resource).filter_by(url=u'http://test_new').one() res_dictized = self.remove_changable_columns(resource_dictize(res, context)) diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index d104c5d3e94..83c1097ad3e 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -192,7 +192,7 @@ def test_18_create_package_not_authorized(self): def test_41_create_resource(self): anna_id = model.Package.by_name(u'annakarenina').id - resource = {'package_id': anna_id, 'url': 'new_url'} + resource = {'package_id': anna_id, 'url': 'http://new_url'} postparams = '%s=1' % json.dumps(resource) res = self.app.post('/api/action/resource_create', params=postparams, @@ -200,7 +200,7 @@ def test_41_create_resource(self): resource = json.loads(res.body)['result'] - assert resource['url'] == 'new_url' + assert resource['url'] == 'http://new_url' def test_42_create_resource_with_error(self): From 528d27bf699b8df85edf8c55f7cb91b8fcf64c7c Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:05:58 +0100 Subject: [PATCH 003/103] [#1678] Make synchronous search the default behaviour Unless already loaded or explicitly disabled via the `ckan.search.automatic_index` configuration option, the synchronous search plugin will be loaded during startup time. --- ckan/config/deployment.ini_tmpl | 7 ++++++- ckan/config/environment.py | 8 ++++++++ doc/configuration.rst | 17 +++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d3c0199687f..d66d6543248 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -24,7 +24,7 @@ app_instance_uuid = ${app_instance_uuid} # List the names of CKAN extensions to activate. # Note: This line is required to be here for packaging, even if it is empty. -ckan.plugins = stats synchronous_search +ckan.plugins = stats # If you'd like to fine-tune the individual locations of the cache data dirs # for the Cache data, or the Session saves, un-comment the desired settings @@ -112,6 +112,11 @@ ckan.gravatar_default = identicon ## Solr support #solr_url = http://127.0.0.1:8983/solr +## Automatic indexing. Make all changes immediately available via the search +## after editing or creating a dataset. Default is true. If for some reason +## you need the indexing to occur asynchronously, set this option to 0. +# ckan.search.automatic_indexing = 1 + ## An 'id' for the site (using, for example, when creating entries in a common search index) ## If not specified derived from the site_url # ckan.site_id = ckan.net diff --git a/ckan/config/environment.py b/ckan/config/environment.py index bb46ea2db8a..8fca75b7548 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -19,6 +19,7 @@ import ckan.lib.search as search import ckan.lib.app_globals as app_globals +log = logging.getLogger(__name__) # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) @@ -122,6 +123,13 @@ def find_controller(self, controller): # load all CKAN plugins p.load_all(config) + # Load the synchronous search plugin, unless already loaded or + # explicitly disabled + if not 'synchronous_search' in config.get('ckan.plugins') and \ + asbool(config.get('ckan.search.automatic_indexing',True)): + log.debug('Loading the synchronous search plugin') + p.load('synchronous_search') + for plugin in p.PluginImplementations(p.IConfigurer): # must do update in place as this does not work: # config = plugin.update_config(config) diff --git a/doc/configuration.rst b/doc/configuration.rst index 9b753e480d9..ebb09845bdc 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -496,6 +496,23 @@ Optionally, ``solr_user`` and ``solr_password`` can also be configured to specif Note, if you change this value, you need to rebuild the search index. +.. index:: + single: ckan.search.automatic_indexing + +ckan.search.automatic_indexing +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.search.automatic_indexing = 1 + +Make all changes immediately available via the search after editing or +creating a dataset. Default is true. If for some reason you need the indexing +to occur asynchronously, set this option to 0. + +Note, this is equivalent to explicitly load the `synchronous_search` plugin. + + simple_search ^^^^^^^^^^^^^ From 67d7135ed8c2494279dbe00b67cfea20ed2bfc6c Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 2 Jul 2012 13:44:37 +0100 Subject: [PATCH 004/103] PEP8 environment.py --- ckan/config/environment.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index 8fca75b7548..d388937ff33 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -24,6 +24,7 @@ # Suppress benign warning 'Unbuilt egg for setuptools' warnings.simplefilter('ignore', UserWarning) + class _Helpers(object): ''' Helper object giving access to template helpers stopping missing functions from causing template exceptions. Useful if @@ -94,13 +95,16 @@ def load_environment(global_conf, app_conf): from pylons.wsgiapp import PylonsApp import pkg_resources find_controller_generic = PylonsApp.find_controller + # This is from pylons 1.0 source, will monkey-patch into 0.9.7 def find_controller(self, controller): if controller in self.controller_classes: return self.controller_classes[controller] # Check to see if its a dotted name if '.' in controller or ':' in controller: - mycontroller = pkg_resources.EntryPoint.parse('x=%s' % controller).load(False) + mycontroller = pkg_resources \ + .EntryPoint \ + .parse('x=%s' % controller).load(False) self.controller_classes[controller] = mycontroller return mycontroller return find_controller_generic(self, controller) @@ -126,7 +130,7 @@ def find_controller(self, controller): # Load the synchronous search plugin, unless already loaded or # explicitly disabled if not 'synchronous_search' in config.get('ckan.plugins') and \ - asbool(config.get('ckan.search.automatic_indexing',True)): + asbool(config.get('ckan.search.automatic_indexing', True)): log.debug('Loading the synchronous search plugin') p.load('synchronous_search') @@ -160,11 +164,13 @@ def find_controller(self, controller): config['pylons.app_globals'] = app_globals.Globals() # add helper functions - restrict_helpers = asbool(config.get('ckan.restrict_template_vars', 'true')) + restrict_helpers = asbool( + config.get('ckan.restrict_template_vars', 'true')) helpers = _Helpers(h, restrict_helpers) config['pylons.h'] = helpers - ## redo template setup to use genshi.search_path (so remove std template setup) + # Redo template setup to use genshi.search_path + # (so remove std template setup) template_paths = [paths['templates'][0]] extra_template_paths = config.get('extra_template_paths', '') if extra_template_paths: @@ -173,6 +179,7 @@ def find_controller(self, controller): # Translator (i18n) translator = Translator(pylons.translator) + def template_loaded(template): translator.setup(template) @@ -203,8 +210,6 @@ def template_loaded(template): # # ################################################################# - - ''' This code is based on Genshi code @@ -273,11 +278,14 @@ def genshi_lookup_attr(cls, obj, key): # Setup the SQLAlchemy database engine # Suppress a couple of sqlalchemy warnings - warnings.filterwarnings('ignore', '^Unicode type received non-unicode bind param value', sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning) - warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning) + msgs = ['^Unicode type received non-unicode bind param value', + "^Did not recognize type 'BIGINT' of column 'size'", + "^Did not recognize type 'tsvector' of column 'search_vector'" + ] + for msg in msgs: + warnings.filterwarnings('ignore', msg, sqlalchemy.exc.SAWarning) - ckan_db = os.environ.get('CKAN_DB') + ckan_db = os.environ.get('CKAN_DB') if ckan_db: config['sqlalchemy.url'] = ckan_db @@ -296,4 +304,3 @@ def genshi_lookup_attr(cls, obj, key): for plugin in p.PluginImplementations(p.IConfigurable): plugin.configure(config) - From b094c4eda6e61cacd3da3e638573803457f014d4 Mon Sep 17 00:00:00 2001 From: Toby Date: Tue, 3 Jul 2012 11:18:48 +0100 Subject: [PATCH 005/103] [#2362] toolkit docs --- doc/index.rst | 1 + doc/toolkit.rst | 148 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 doc/toolkit.rst diff --git a/doc/index.rst b/doc/index.rst index e8973a28272..ccd7a8d6dae 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -36,6 +36,7 @@ Customizing and Extending commenting extensions writing-extensions + toolkit forms tag-vocabularies form-integration diff --git a/doc/toolkit.rst b/doc/toolkit.rst new file mode 100644 index 00000000000..29ba543ab00 --- /dev/null +++ b/doc/toolkit.rst @@ -0,0 +1,148 @@ + +Plugins Toolkit +=============== + +To allow a safe way for extensions to interact with ckan a toolkit is +provided. We aim to keep this toolkit stable across ckan versions so +that extensions will work across diferent versions of ckan. + +.. Note:: + + It is advised that when writing extensions that all interaction with + ckan is done via the toolkit so that they do not break when new + versions of ckan are released. + +Over time we will be expanding the functionality available via +this toolkit. + +Example extension that registers a new helper function available to +templates via h.example_helper() :: + + import ckan.plugins as p + + + class ExampleExtension(p.SingletonPlugin): + + p.implements(p.IConfigurer) + p.implements(p.ITemplateHelpers) + + def update_config(self, config): + # add template directory that contains our snippet + p.toolkit.add_template_directory(config, 'templates') + + @classmethod + def example_helper(cls, data=None): + # render our custom snippet + return p.toolkit.render_snippet('custom_snippet.html', data) + + + def get_helpers(self): + # register our helper function + return {'example_helper': self.example_helper} + +The following functions, classes and exceptions are provided by the toolkit. + +*class* **CkanCommand** + Base class for building paster functions. + + +*exception* **CkanVersionException** + Exception raised if required ckan version is not available. + + +*exception* **NotAuthorized** + Exception raised when an action is not permitted by a user. + + +*exception* **ObjectNotFound** + Exception raised when an object cannot be found. + + +*exception* **ValidationError** + Exception raised when supplied data is invalid. + it contains details of the error that occurred. + + +**_** (*value*) + Mark a string for translation. Returns the localized unicode + string of value. + + Mark a string to be localized as follows:: + + _('This should be in lots of languages') + + + + +**add_public_directory** (*config, relative_path*) + Function to aid adding extra public paths to the config. + The path is relative to the file calling this function. + + +**add_template_directory** (*config, relative_path*) + Function to aid adding extra template paths to the config. + The path is relative to the file calling this function. + + +**asbool** (*obj*) + part of paste.deploy.converters: convert strings like yes, no, true, false, 0, 1 to boolean + + +**asint** (*obj*) + part of paste.deploy.converters: convert stings to integers + + +**aslist** (*obj, sep=None, strip=True*) + part of paste.deploy.converters: convert string objects to a list + + +**check_access** (*action, context, data_dict=None*) + check that the named action with the included context and + optional data dict is allowed raises NotAuthorized if the action is + not permitted or True. + + +**check_ckan_version** (*min_version=None, max_version=None*) + Check that the ckan version is correct for the plugin. + + +**get_action** (*action*) + Get the requested action function. + + +*class* **literal** + Represents an HTML literal. + + This subclass of unicode has a ``.__html__()`` method that is + detected by the ``escape()`` function. + + Also, if you add another string to this string, the other string + will be quoted and you will get back another literal object. Also + ``literal(...) % obj`` will quote any value(s) from ``obj``. If + you do something like ``literal(...) + literal(...)``, neither + string will be changed because ``escape(literal(...))`` doesn't + change the original literal. + + Changed in WebHelpers 1.2: the implementation is now now a subclass of + ``markupsafe.Markup``. This brings some new methods: ``.escape`` (class + method), ``.unescape``, and ``.striptags``. + + + + +**render** (*template_name, data=None*) + Main template render function. + + +**render_snippet** (*template_name, data=None*) + helper for the render_snippet function + similar to the render function. + + +**render_text** (*template_name, data=None*) + Render genshi text template. + + +**requires_ckan_version** (*min_version, max_version=None*) + Check that the ckan version is correct for the plugin. + From 9b72e88919fcdd25c0156a430f00968c9ec76ec3 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:02:30 +0200 Subject: [PATCH 006/103] [#2627] Refactor form_to_db_schema_options Make it call self.form_to_db_schema so that the form_to_db_schema() methods of IDatasetForm extensions get called. Also make it call new form_to_db_schema_api_create() and form_to_db_schema_api_update() methods which could potentially be overridden by extensions also. --- ckan/lib/plugins.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 258e1449681..4ce1f4236ca 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -221,11 +221,21 @@ def form_to_db_schema_options(self, options): if options.get('api'): if options.get('type') == 'create': - return logic.schema.default_create_package_schema() + return self.form_to_db_schema_api_create() else: - return logic.schema.default_update_package_schema() + assert options.get('type') == 'update' + return self.form_to_db_schema_api_update() else: - return logic.schema.package_form_schema() + return self.form_to_db_schema() + + def form_to_db_schema(self, options): + return logic.schema.package_form_schema() + + def form_to_db_schema_api_create(self): + return logic.schema.default_create_package_schema() + + def form_to_db_schema_api_update(self): + return logic.schema.default_update_package_schema() def db_to_form_schema(self): '''This is an interface to manipulate data from the database From d93cb58049bf79a1983f3b96759361c0627684a1 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:32:00 +0200 Subject: [PATCH 007/103] Fix a crasher in package/read.html template Sometimes c.pkg_dict has no member named groups (e.g. when there is an active IDatasetForm plugin with a db_to_form_schema() method). --- ckan/templates/package/read.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/read.html b/ckan/templates/package/read.html index 29a03720bbd..7c76ef39192 100644 --- a/ckan/templates/package/read.html +++ b/ckan/templates/package/read.html @@ -45,7 +45,7 @@

Tags

${tag_list(c.pkg_dict.get('tags', ''))} -
  • +

  • Groups

    • From 6f91aa595b74e3ff4af2128cef11f76786f8ef20 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 14:51:11 +0200 Subject: [PATCH 008/103] Add db_to_form_schema method For IDatasetForm plugins to base their db_to_form_schema() dicts on --- ckan/logic/schema.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 8257408bb68..301f46b0b8b 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,6 +175,23 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema +def db_to_form_schema(): + schema = default_package_schema() + # Workaround a bug in CKAN's convert_from_tags() function. + # TODO: Fix this issue in convert_from_tags(). + schema.update({ + 'tags': { + '__extras': [ckan.lib.navl.validators.keep_extras, + ckan.logic.converters.free_tags_only] + }, + }) + # Workaround a bug in CKAN. + # TODO: Fix this elsewhere so we don't need to workaround it here. + schema['resources'].update({ + 'created': [ckan.lib.navl.validators.ignore_missing], + }) + return schema + def default_group_schema(): schema = { From 126c94b730b9c399dc2ba9c1a5c8858d7d97b802 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:02:05 +0200 Subject: [PATCH 009/103] Rename db_to_form_schema -> db_to_form_package_schema() --- ckan/logic/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 301f46b0b8b..5ae86675633 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -175,7 +175,7 @@ def package_form_schema(): schema.pop('relationships_as_subject') return schema -def db_to_form_schema(): +def db_to_form_package_schema(): schema = default_package_schema() # Workaround a bug in CKAN's convert_from_tags() function. # TODO: Fix this issue in convert_from_tags(). From 786479e8e6eea392fc4a4312c7c8a534a08083a8 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:20:14 +0200 Subject: [PATCH 010/103] Rename package_form_schema() -> form_to_db_package_schema() The only purpose of this schema is to be used as a base schema by IDatasetForm plugins for their form_to_db_schema() methods, hopefully the new name makes that clearer --- ckan/lib/plugins.py | 6 +++--- ckan/logic/schema.py | 2 +- ckan/tests/schema/test_schema.py | 2 +- ckanext/publisher_form/forms.py | 1 - ckanext/test_tag_vocab_plugin.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ckan/lib/plugins.py b/ckan/lib/plugins.py index 4ce1f4236ca..bb5b795aa18 100644 --- a/ckan/lib/plugins.py +++ b/ckan/lib/plugins.py @@ -228,8 +228,8 @@ def form_to_db_schema_options(self, options): else: return self.form_to_db_schema() - def form_to_db_schema(self, options): - return logic.schema.package_form_schema() + def form_to_db_schema(self): + return logic.schema.form_to_db_package_schema() def form_to_db_schema_api_create(self): return logic.schema.default_create_package_schema() @@ -397,7 +397,7 @@ def check_data_dict(self, data_dict): 'extras_validation', 'save', 'return_to', 'resources'] - schema_keys = package_form_schema().keys() + schema_keys = form_to_db_package_schema().keys() keys_in_schema = set(schema_keys) - set(surplus_keys_schema) missing_keys = keys_in_schema - set(data_dict.keys()) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 5ae86675633..ba4f9472afb 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,7 +152,7 @@ def default_update_package_schema(): return schema -def package_form_schema(): +def form_to_db_package_schema(): schema = default_package_schema() ##new diff --git a/ckan/tests/schema/test_schema.py b/ckan/tests/schema/test_schema.py index e8d67e2c547..ab2c3d629d6 100644 --- a/ckan/tests/schema/test_schema.py +++ b/ckan/tests/schema/test_schema.py @@ -94,7 +94,7 @@ def test_tag_string_parsing(self): # errors correctly. context = {'model': ckan.model, 'session': ckan.model.Session} - schema = ckan.logic.schema.package_form_schema() + schema = ckan.logic.schema.form_to_db_package_schema() # basic parsing of comma separated values tests = (('tag', ['tag'], []), diff --git a/ckanext/publisher_form/forms.py b/ckanext/publisher_form/forms.py index 672209f6b4a..0d23d7aed10 100644 --- a/ckanext/publisher_form/forms.py +++ b/ckanext/publisher_form/forms.py @@ -9,7 +9,6 @@ from ckan.logic import tuplize_dict, clean_dict, parse_params import ckan.logic.schema as default_schema from ckan.logic.schema import group_form_schema -from ckan.logic.schema import package_form_schema import ckan.logic.validators as val from ckan.lib.base import BaseController, render, c, model, abort, request from ckan.lib.base import redirect, _, config, h diff --git a/ckanext/test_tag_vocab_plugin.py b/ckanext/test_tag_vocab_plugin.py index 043234890b2..2fe95f4ed9d 100644 --- a/ckanext/test_tag_vocab_plugin.py +++ b/ckanext/test_tag_vocab_plugin.py @@ -7,7 +7,7 @@ from genshi.filters import Transformer from ckan.logic import get_action from ckan.logic.converters import convert_to_tags, convert_from_tags, free_tags_only -from ckan.logic.schema import package_form_schema, default_package_schema +from ckan.logic.schema import default_package_schema from ckan.lib.navl.validators import ignore_missing, keep_extras from ckan import plugins From 74b346ee9a2bbbc48021ffb2e3ead6fbd5923654 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Fri, 6 Jul 2012 15:32:51 +0200 Subject: [PATCH 011/103] Add back in deprectated package_form_schema() for backwards compatibility --- ckan/logic/schema.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index ba4f9472afb..2bb0b58b3ac 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -152,6 +152,11 @@ def default_update_package_schema(): return schema +def package_form_schema(): + # This function is deprecated and was replaced by + # form_to_db_package_schema(), it remains here for backwards compatibility. + return form_to_db_package_schema() + def form_to_db_package_schema(): schema = default_package_schema() From 9063f7be8bf73f5a85ee527838c27056facce6a4 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 13:43:59 +0100 Subject: [PATCH 012/103] [#2313] Remove deprecated facet_items helper. --- ckan/lib/helpers.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 114efb0a78e..0063c9618df 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -377,34 +377,6 @@ def unselected_facet_items(facet, limit=10): facets.append(facet_item) return sorted(facets, key=lambda item: item['count'], reverse=True)[:limit] -@deprecated() -def facet_items(*args, **kwargs): - """ - DEPRECATED: Use the new facet data structure, and `unselected_facet_items()` - """ - _log.warning('Deprecated function: ckan.lib.helpers:facet_items(). Will be removed in v1.8') - # facet_items() used to need c passing as the first arg - # this is deprecated as pointless - # throws error if ckan.restrict_template_vars is True - # When we move to strict helpers then this should be removed as a wrapper - if len(args) > 2 or (len(args) > 0 and 'name' in kwargs) or (len(args) > 1 and 'limit' in kwargs): - if not asbool(config.get('ckan.restrict_template_vars', 'false')): - return _facet_items(*args[1:], **kwargs) - raise Exception('facet_items() calling has been changed. remove c in template %s or included one' % c.__template_name) - return _facet_items(*args, **kwargs) - - -def _facet_items(name, limit=10): - if not c.facets or not c.facets.get(name): - return [] - facets = [] - for k, v in c.facets.get(name).items(): - if not len(k.strip()): - continue - if not (name, k) in request.params.items(): - facets.append((k, v)) - return sorted(facets, key=lambda (k, v): v, reverse=True)[:limit] - def facet_title(name): return config.get('search.facets.%s.title' % name, name.capitalize()) From 04a9fd7be98675eeb472e6695234c36cb2d6674a Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 13:55:12 +0100 Subject: [PATCH 013/103] [#2313] Removed deprecated facet_sidebar() function --- ckan/templates/facets.html | 54 ------------------- .../templates/publisher_read.html | 4 +- 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/ckan/templates/facets.html b/ckan/templates/facets.html index 001be8c79c3..939f3ee281b 100644 --- a/ckan/templates/facets.html +++ b/ckan/templates/facets.html @@ -75,60 +75,6 @@

      ${h.facet_title(title)}

      • - - - -
      -

      ${title(code)}

      - -

      ${if_empty}

      -
      -       - - - - -
    • ${if_empty}
      • -
    • - ${label(facet_item.name)} (facet_item['count']) -
      • -       -
      From aaff2a1175a4ecb16c272378c8facfedd6a56412 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sat, 7 Jul 2012 15:03:37 +0100 Subject: [PATCH 015/103] [#2313] Remove c.facets. It's been deprecated by c.search_facets. --- ckan/controllers/group.py | 2 -- ckan/controllers/home.py | 7 ++++--- ckan/controllers/package.py | 2 -- ckan/logic/action/get.py | 3 +-- ckan/tests/logic/test_action.py | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 8bb6119f513..41bec6ee26f 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -200,13 +200,11 @@ def pager_url(q=None, page=None): item_count=query['count'], items_per_page=limit ) - c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True - c.facets = {} c.page = h.Page(collection=[]) # Add the group's activity stream (already rendered to HTML) to the diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index e885f3bda9e..79e1d1ec889 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -49,13 +49,14 @@ def index(self): query = ckan.logic.get_action('package_search')( context, data_dict) c.package_count = query['count'] - c.facets = query['facets'] + c.search_facets = query['search_facets'] data_dict = {'order_by': 'packages', 'all_fields': 1} # only give the terms to group dictize that are returned in the # facets as full results take a lot longer - if 'groups' in c.facets: - data_dict['groups'] = c.facets['groups'].keys() + if 'groups' in c.search_facets: + data_dict['groups'] = [ item['name'] for item in + c.search_facets['groups']['items'] ] c.groups = ckan.logic.get_action('group_list')(context, data_dict) except SearchError, se: c.package_count = 0 diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index ccab6931135..218c746f7db 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -216,13 +216,11 @@ def pager_url(q=None, page=None): item_count=query['count'], items_per_page=limit ) - c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True - c.facets = {} c.page = h.Page(collection=[]) return render(self._search_template(package_type)) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index acdab602e75..460aca6f3b7 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1137,13 +1137,12 @@ def package_search(context, data_dict): search_results = { 'count': count, - 'facets': facets, 'results': results } # Transform facets into a more useful data structure. restructured_facets = {} - for key, value in search_results['facets'].items(): + for key, value in facets.items(): restructured_facets[key] = { 'title': key, 'items': [] diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 3b3566fecf7..3aa97bd91f9 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -1318,7 +1318,7 @@ def after_search(self, search_results, search_params): assert 'results' in search_results assert 'count' in search_results - assert 'facets' in search_results + assert 'search_facets' in search_results if 'extras' in search_params and 'ext_avoid' in search_params['extras']: # Remove results with a certain value From 73f839b31b0c17e97190c0ab01ae72b86d473249 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Sun, 8 Jul 2012 10:25:17 +0100 Subject: [PATCH 016/103] [#2313] Add changes to changelog --- CHANGELOG.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3a117515ddf..ad9c5dd9feb 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -24,6 +24,12 @@ v1.8 API has been rewritten, with each function and its arguments and return values now individually documented. +API changes and deprecation: +* [#2313] Deprecated functions related to the old faceting data structure have + been removed. `helpers.py:facet_items()`, `facets.html:facet_sidebar()`, + `facets.html:facet_list_items()` and the context is no longer populated with + `facets` field, use `search_facets` instead. + v1.7.1 2012-06-20 ================= From 3a518547e18a1b9d2177db9243620272c58f3ba9 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 9 Jul 2012 09:43:38 +0100 Subject: [PATCH 017/103] Fixes 2624, where tags were not saved with organization_dataset Added tag_convert_string to the validators for the form_to_db_schema so that it correctly converted the string to tags in the DB. --- ckanext/organizations/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckanext/organizations/forms.py b/ckanext/organizations/forms.py index caf9226c2da..3e22ff4e297 100644 --- a/ckanext/organizations/forms.py +++ b/ckanext/organizations/forms.py @@ -11,7 +11,7 @@ from ckan.plugins import IGroupForm, IDatasetForm, IConfigurer, IRoutes from ckan.plugins import implements, SingletonPlugin from ckan.logic import check_access - +from ckan.logic.validators import tag_string_convert from ckan.lib.navl.validators import (ignore_missing, not_empty, empty, @@ -237,6 +237,7 @@ def form_to_db_schema(self): schema = default_package_schema() schema['groups']['capacity'] = [ignore_missing, unicode] schema['__after'] = [group_required] + schema['tag_string'] = [ignore_missing, tag_string_convert] return schema def check_data_dict(self, data_dict, schema=None): From 4f32c46e50d2c9767faf6768ac013248d83d8d82 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 9 Jul 2012 12:37:32 +0200 Subject: [PATCH 018/103] [#2624] Base organizations form_to_db_schema() off form_to_db_package_schema() --- ckanext/organizations/forms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckanext/organizations/forms.py b/ckanext/organizations/forms.py index 3e22ff4e297..5e33eaa8f24 100644 --- a/ckanext/organizations/forms.py +++ b/ckanext/organizations/forms.py @@ -3,7 +3,7 @@ import ckan.authz as authz from ckan.logic import NotAuthorized -from ckan.logic.schema import group_form_schema, default_package_schema +from ckan.logic.schema import group_form_schema, form_to_db_package_schema from ckan.lib import base from ckan.lib.base import c, model, abort, request from ckan.lib.base import redirect, _, config, h @@ -234,10 +234,9 @@ def db_to_form_schema(self): into a format suitable for the form (optional)''' def form_to_db_schema(self): - schema = default_package_schema() + schema = form_to_db_package_schema() schema['groups']['capacity'] = [ignore_missing, unicode] schema['__after'] = [group_required] - schema['tag_string'] = [ignore_missing, tag_string_convert] return schema def check_data_dict(self, data_dict, schema=None): From 3bae84bd1352d4a2b9fe37579f7cfcf87fef49ab Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 9 Jul 2012 11:57:20 +0100 Subject: [PATCH 019/103] [#2634] add restricted template vars to change log --- CHANGELOG.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3a117515ddf..e1eabee6a85 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -23,6 +23,13 @@ v1.8 * [#2345] New action API reference docs. The documentation for CKAN's Action API has been rewritten, with each function and its arguments and return values now individually documented. +* Template helper functions are now restricted by default. This is + part of a process of refactoring the CKAN code. By default only + those helper functions listed in lib.helpers.__allowed_functions__ + are available to templates. The full functions can still be made + available by `setting ckan.restrict_template_vars = false` in your + .ini config file. Only restricted functions will be allowed in + future versions of CKAN. v1.7.1 2012-06-20 ================= From b41b57c5fb686c76a6a1a1dd20df5a358b3c7a93 Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 9 Jul 2012 12:45:27 +0100 Subject: [PATCH 020/103] [#2632] check allowed helper functions exist --- ckan/config/environment.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index d388937ff33..f07bb1d979a 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -32,7 +32,7 @@ class _Helpers(object): not been enabled. ''' def __init__(self, helpers, restrict=True): functions = {} - allowed = helpers.__allowed_functions__ + allowed = helpers.__allowed_functions__[:] # list of functions due to be deprecated self.deprecated = [] @@ -42,8 +42,13 @@ def __init__(self, helpers, restrict=True): if restrict: continue functions[helper] = getattr(helpers, helper) + allowed.remove(helper) self.functions = functions + if allowed: + raise Exception('Template helper function(s) `%s` not defined' + % ', '.join(allowed)) + # extend helper functions with ones supplied by plugins extra_helpers = [] for plugin in p.PluginImplementations(p.ITemplateHelpers): From f6da5124e79d5aad6c14b5e4451e5eae07fdac9e Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 Jul 2012 15:03:51 +0100 Subject: [PATCH 021/103] Change default authorization settings These are the new default authotization behaviours: * Only logged in users are able to create datasets and upload files * Only the user that created the dataset can edit it by default NB: this will break a large number of test. They will be fixed on following commits. --- ckan/config/deployment.ini_tmpl | 4 ++-- ckan/model/authz.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ckan/config/deployment.ini_tmpl b/ckan/config/deployment.ini_tmpl index d3c0199687f..5d9b4e4826c 100644 --- a/ckan/config/deployment.ini_tmpl +++ b/ckan/config/deployment.ini_tmpl @@ -141,9 +141,9 @@ ckan.dump_dir = %(here)s/dump ckan.backup_dir = %(here)s/backup # Default authorizations for new domain objects -#ckan.default_roles.Package = {"visitor": ["editor"], "logged_in": ["editor"]} +#ckan.default_roles.Package = {"visitor": ["reader"], "logged_in": ["reader"]} #ckan.default_roles.Group = {"visitor": ["reader"], "logged_in": ["reader"]} -#ckan.default_roles.System = {"visitor": ["anon_editor"], "logged_in": ["editor"]} +#ckan.default_roles.System = {"visitor": ["reader"], "logged_in": ["editor"]} #ckan.default_roles.AuthorizationGroup = {"visitor": ["reader"], "logged_in": ["reader"]} ## Ckan public and private recaptcha keys [localhost] diff --git a/ckan/model/authz.py b/ckan/model/authz.py index c95af3e43a5..7c952760ba4 100644 --- a/ckan/model/authz.py +++ b/ckan/model/authz.py @@ -406,9 +406,9 @@ def give_all_packages_default_user_roles(): # default user roles - used when the config doesn\'t specify them default_default_user_roles = { - 'Package': {"visitor": ["editor"], "logged_in": ["editor"]}, + 'Package': {"visitor": ["reader"], "logged_in": ["reader"]}, 'Group': {"visitor": ["reader"], "logged_in": ["reader"]}, - 'System': {"visitor": ["anon_editor"], "logged_in": ["editor"]}, + 'System': {"visitor": ["reader"], "logged_in": ["editor"]}, 'AuthorizationGroup': {"visitor": ["reader"], "logged_in": ["reader"]}, } From 2b0932a105d9c1292814b2afac4bf237a8db3008 Mon Sep 17 00:00:00 2001 From: Toby Date: Mon, 9 Jul 2012 15:42:24 +0100 Subject: [PATCH 022/103] [#2638] get_request_param() helper function --- ckan/lib/helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 114efb0a78e..586355c0205 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -885,6 +885,11 @@ def dashboard_activity_stream(user_id): context = {'model' : model, 'session':model.Session, 'user':c.user} return logic.get_action('dashboard_activity_list_html')(context, {'id': user_id}) +def get_request_param(parameter_name, default=None): + ''' This function allows templates to access query string parameters + from the request. This is useful for things like sort order in + searches. ''' + return request.params.get(parameter_names, default) # these are the functions that will end up in `h` template helpers # if config option restrict_template_vars is true @@ -942,6 +947,7 @@ def dashboard_activity_stream(user_id): 'follow_button', 'follow_count', 'dashboard_activity_stream', + 'get_request_param', # imported into ckan.lib.helpers 'literal', 'link_to', From b55c85e4fbc975c4e46dad1c15151ede3f0a6b06 Mon Sep 17 00:00:00 2001 From: amercader Date: Mon, 9 Jul 2012 16:37:55 +0100 Subject: [PATCH 023/103] Fix broken tests after default auth settings change All tests involving non logged-in editing were failing, as this is no longer allowed. The changes on auth settings basically mean than everytime some editing is tested, a valid user with permissions on the dataset must be provided, e.g.: offset = url_for(controller='package', action='edit', id=myid) self.app.get(offset,extra_environ={'REMOTE_USER': 'russianfan'}) or: res = self.app.post('/api/action/resource_create', params=postparams, extra_environ={'Authorization': api_key }) --- ckan/tests/forms/test_authz.py | 8 +- ckan/tests/functional/api/test_activity.py | 110 +++++++------ ckan/tests/functional/test_admin.py | 4 +- ckan/tests/functional/test_authz.py | 2 +- ckan/tests/functional/test_home.py | 3 +- ckan/tests/functional/test_package.py | 149 ++++++++---------- .../functional/test_package_edit_authz.py | 67 +++----- ckan/tests/functional/test_search.py | 7 - ckan/tests/functional/test_storage.py | 16 +- ckan/tests/functional/test_upload.py | 5 +- ckan/tests/logic/test_action.py | 7 +- ckan/tests/models/test_authz.py | 22 ++- ckan/tests/models/test_repo.py | 4 +- 13 files changed, 203 insertions(+), 201 deletions(-) diff --git a/ckan/tests/forms/test_authz.py b/ckan/tests/forms/test_authz.py index 3b02b00ed0d..f7d31c143db 100644 --- a/ckan/tests/forms/test_authz.py +++ b/ckan/tests/forms/test_authz.py @@ -47,11 +47,11 @@ def teardown_class(self): def test_0_no_change(self): pkg_name, user = self._new_pkg(0) - before_roles = { 'visitor': 'editor', - 'logged_in': 'editor', + before_roles = { 'visitor': 'reader', + 'logged_in': 'reader', user.name: 'admin' } - after_roles = { 'visitor': 'editor', - 'logged_in': 'editor', + after_roles = { 'visitor': 'reader', + 'logged_in': 'reader', user.name: 'admin' } num_prs_before = model.Session.query(model.PackageRole).filter_by(user_id=user.id).count() self._test_change(before_roles, after_roles, pkg_name) diff --git a/ckan/tests/functional/api/test_activity.py b/ckan/tests/functional/api/test_activity.py index 4662909ca88..da34969e60a 100644 --- a/ckan/tests/functional/api/test_activity.py +++ b/ckan/tests/functional/api/test_activity.py @@ -2,6 +2,8 @@ import logging logger = logging.getLogger(__name__) +from nose.plugins.skip import SkipTest + import ckan import ckan.model as model from ckan.logic.action.create import package_create as _package_create @@ -1302,21 +1304,25 @@ def test_01_update_extras_not_logged_in(self): when a package extra is changed by a user who is not logged in. """ - context = { - 'model': model, - 'session': model.Session, - 'user': self.normal_user.name, - 'extras_as_string': True, - } - packages_with_extras = [] - for package_name in package_list(context, {}): - package_dict = package_show(context, {'id': package_name}) - if len(package_dict['extras']) > 0: - packages_with_extras.append(package_dict) - assert len(packages_with_extras) > 0, ( - "Need some packages with extras to test") - for package_dict in packages_with_extras: - self._update_extra(package_dict, None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# context = { +# 'model': model, +# 'session': model.Session, +# 'user': self.normal_user.name, +# 'extras_as_string': True, +# } +# packages_with_extras = [] +# for package_name in package_list(context, {}): +# package_dict = package_show(context, {'id': package_name}) +# if len(package_dict['extras']) > 0: +# packages_with_extras.append(package_dict) +# assert len(packages_with_extras) > 0, ( +# "Need some packages with extras to test") +# for package_dict in packages_with_extras: +# self._update_extra(package_dict, None) +# def test_01_update_package(self): """ @@ -1337,8 +1343,11 @@ def test_01_update_package_not_logged_in(self): when packages are updated by a user who is not logged in. """ - for package in model.Session.query(model.Package).all(): - self._update_package(package, user=None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# for package in model.Session.query(model.Package).all(): +# self._update_package(package, user=None) def test_01_update_resource(self): """ @@ -1386,7 +1395,10 @@ def test_create_package_not_logged_in(self): when a new package is created by a user who is not logged in. """ - self._create_package(user=None, name="not_logged_in_test_package") + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# self._create_package(user=None, name="not_logged_in_test_package") def test_add_resources(self): """ @@ -1407,8 +1419,11 @@ def test_add_resources_not_logged_in(self): when a resource is added to a package by a user who is not logged in. """ - for package in model.Session.query(model.Package).all(): - self._add_resource(package, user=None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# for package in model.Session.query(model.Package).all(): +# self._add_resource(package, user=None) def test_delete_package(self): """ @@ -1970,16 +1985,19 @@ def test_add_extras_not_logged_in(self): when an extra is added to a package by a user who is not logged in. """ - context = { - 'model': model, - 'session': model.Session, - 'user': self.normal_user.name, - 'extras_as_string': True, - } - for package_name in package_list(context, {}): - package_dict = package_show(context, {'id': package_name}) - self._add_extra(package_dict, None, key='not_logged_in_extra_key') + raise SkipTest +## TODO: remove, non logged in users can no longer edit packages +# context = { +# 'model': model, +# 'session': model.Session, +# 'user': self.normal_user.name, +# 'extras_as_string': True, +# } +# for package_name in package_list(context, {}): +# package_dict = package_show(context, {'id': package_name}) +# self._add_extra(package_dict, None, key='not_logged_in_extra_key') +# def test_delete_extras(self): """ Test deleted package extra activity stream. @@ -2012,21 +2030,25 @@ def test_delete_extras_not_logged_in(self): when a package extra is deleted by a user who is not logged in. """ - context = { - 'model': model, - 'session': model.Session, - 'user': self.normal_user.name, - 'extras_as_string': True, - } - packages_with_extras = [] - for package_name in package_list(context, {}): - package_dict = package_show(context, {'id': package_name}) - if len(package_dict['extras']) > 0: - packages_with_extras.append(package_dict) - assert len(packages_with_extras) > 0, ( - "Need some packages with extras to test") - for package_dict in packages_with_extras: - self._delete_extra(package_dict, None) + raise SkipTest + +## TODO: remove, non logged in users can no longer edit packages +# context = { +# 'model': model, +# 'session': model.Session, +# 'user': self.normal_user.name, +# 'extras_as_string': True, +# } +# packages_with_extras = [] +# for package_name in package_list(context, {}): +# package_dict = package_show(context, {'id': package_name}) +# if len(package_dict['extras']) > 0: +# packages_with_extras.append(package_dict) +# assert len(packages_with_extras) > 0, ( +# "Need some packages with extras to test") +# for package_dict in packages_with_extras: +# self._delete_extra(package_dict, None) +# def test_follow_dataset(self): user = self.normal_user diff --git a/ckan/tests/functional/test_admin.py b/ckan/tests/functional/test_admin.py index 416f1b0b548..ad5bf3f21cb 100644 --- a/ckan/tests/functional/test_admin.py +++ b/ckan/tests/functional/test_admin.py @@ -109,7 +109,7 @@ def authz_submit(form): # before we start changing things, check that the roles on the system are as expected assert original_user_roles == \ - [(u'logged_in', u'editor'), (u'testsysadmin', u'admin'), (u'visitor', u'anon_editor')] , \ + [(u'logged_in', u'editor'), (u'testsysadmin', u'admin'), (u'visitor', u'reader')] , \ "original user roles not as expected " + str(original_user_roles) assert original_authzgroup_roles == [(u'anauthzgroup', u'editor')], \ @@ -160,7 +160,7 @@ def authz_submit(form): # change lots of things form = get_user_form() check_and_set_checkbox(form, u'visitor', u'editor', False, True) - check_and_set_checkbox(form, u'visitor', u'reader', False, False) + check_and_set_checkbox(form, u'visitor', u'reader', True, False) check_and_set_checkbox(form, u'logged_in', u'editor', True, False) check_and_set_checkbox(form, u'logged_in', u'reader', False, True) submit(form) diff --git a/ckan/tests/functional/test_authz.py b/ckan/tests/functional/test_authz.py index 6b2e55dd62b..bd8bcb9f11e 100644 --- a/ckan/tests/functional/test_authz.py +++ b/ckan/tests/functional/test_authz.py @@ -342,7 +342,7 @@ def test_02_visitor_edits(self): self._test_can('edit', self.visitor, [], interfaces=['rest']) def test_visitor_creates(self): - self._test_can('create', self.visitor, [], interfaces=['wui'], entity_types=['dataset']) + self._test_cant('create', self.visitor, [], interfaces=['wui'], entity_types=['dataset']) self._test_cant('create', self.visitor, [], interfaces=['wui'], entity_types=['group']) # need to be sysadmin self._test_cant('create', self.visitor, [], interfaces=['rest']) diff --git a/ckan/tests/functional/test_home.py b/ckan/tests/functional/test_home.py index f47893dbb48..f69ca90d5da 100644 --- a/ckan/tests/functional/test_home.py +++ b/ckan/tests/functional/test_home.py @@ -25,7 +25,8 @@ def teardown_class(self): def test_home_page(self): offset = url_for('home') res = self.app.get(offset) - assert 'Add a dataset' in res + # Non logged in users can no longer add datasets + assert not 'Add a dataset' in res assert 'Could not change language' not in res assert "Dave's books has 2 datasets" in res, res assert "Roger's books has 1 datasets" in res, res diff --git a/ckan/tests/functional/test_package.py b/ckan/tests/functional/test_package.py index ffe423737fb..59911a2bc06 100644 --- a/ckan/tests/functional/test_package.py +++ b/ckan/tests/functional/test_package.py @@ -199,7 +199,7 @@ def check_form_filled_correctly(self, res, **params): assert params['log_message'] in main_res, main_res def _check_redirect(self, return_url_param, expected_redirect, - pkg_name_to_edit=''): + pkg_name_to_edit='',extra_environ=None): ''' @param return_url_param - encoded url to be given as param - if None then assume redirect is specified in pylons config @@ -223,12 +223,12 @@ def _check_redirect(self, return_url_param, expected_redirect, if return_url_param: offset_params['return_to'] = return_url_param offset = url_for(**offset_params) - res = self.app.get(offset) + res = self.app.get(offset, extra_environ=extra_environ) assert 'Datasets -' in res fv = res.forms['dataset-edit'] prefix = '' fv[prefix + 'name'] = new_name - res = fv.submit('save', status=302) + res = fv.submit('save', status=302, extra_environ=extra_environ) assert not 'Error' in res, res redirected_to = dict(res.headers).get('Location') or dict(res.headers)['location'] expected_redirect_url = expected_redirect.replace('', new_name) @@ -298,7 +298,6 @@ def test_read(self): assert 'russian' in res assert 'david' in res assert 'roger' in res - assert 'State' in res assert 'genre' in res, res assert 'romantic novel' in res, res assert 'original media' in res, res @@ -574,11 +573,12 @@ class TestEdit(TestPackageForm): @classmethod def setup_class(self): CreateTestData.create() + self._reset_data() def setup(self): if not self.res: - self.res = self.app.get(self.offset) + self.res = self.app.get(self.offset,extra_environ=self.extra_environ_admin) model.Session.remove() @classmethod @@ -602,6 +602,9 @@ def _reset_data(self): self.editpkg = model.Package.by_name(self.editpkg_name) self.admin = model.User.by_name(u'testadmin') + + self.extra_environ_admin = {'REMOTE_USER': self.admin.name.encode('utf8')} + self.extra_environ_russianfan = {'REMOTE_USER': 'russianfan'} self.res = None #get's refreshed by setup model.Session.remove() @@ -612,7 +615,7 @@ def teardown_class(self): def test_edit_basic(self): # just the absolute basics try: - self.res = self.app.get(self.offset) + self.res = self.app.get(self.offset,extra_environ=self.extra_environ_admin) assert 'Edit - Datasets' in self.res, self.res new_name = u'new-name' new_title = u'New Title' @@ -620,7 +623,7 @@ def test_edit_basic(self): prefix = '' fv[prefix + 'name'] = new_name fv[prefix + 'title'] = new_title - res = fv.submit('save') + res = fv.submit('save',extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() offset = url_for(controller='package', action='read', id=new_name) @@ -635,7 +638,7 @@ def test_edit_basic(self): def test_edit(self): # just the key fields try: - self.res = self.app.get(self.offset) + self.res = self.app.get(self.offset,extra_environ=self.extra_environ_admin) assert 'Edit - Datasets' in self.res, self.res assert self.editpkg.notes in self.res @@ -653,7 +656,7 @@ def test_edit(self): #fv[prefix + 'resources__0__url'] = new_download_url fv[prefix + 'license_id'] = newlicense_id fv[prefix + 'version'] = newversion - res = fv.submit('save') + res = fv.submit('save',extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() model.Session.remove() @@ -673,7 +676,7 @@ def test_edit_basic_pkg_by_id(self): try: pkg = model.Package.by_name(u'editpkgtest') offset = url_for(controller='package', action='edit', id=pkg.id) - res = self.app.get(offset) + res = self.app.get(offset, extra_environ=self.extra_environ_admin) #assert res.body == self.res.body, self.diff_responses(res, self.res) assert 'Edit - Datasets' in res, res assert pkg.name in res @@ -683,7 +686,7 @@ def test_edit_basic_pkg_by_id(self): prefix = '' fv[prefix + 'name'] = new_name fv[prefix + 'title'] = new_title - res = fv.submit('save') + res = fv.submit('save', extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() offset = url_for(controller='package', action='read', id=new_name) @@ -710,7 +713,7 @@ def test_edit_2_tags_and_groups(self): fv[prefix + 'tag_string'] = tagvalues exp_log_message = 'test_edit_2: making some changes' fv['log_message'] = exp_log_message - res = fv.submit('save') + res = fv.submit('save', extra_environ=self.extra_environ_admin) # get redirected ... res = res.follow() assert '%s - Datasets' % self.editpkg_name in res @@ -720,7 +723,7 @@ def test_edit_2_tags_and_groups(self): for tag in newtags: assert tag in outtags rev = model.Revision.youngest(model.Session) - assert rev.author == 'Unknown IP Address' + assert rev.author == self.admin.name assert rev.message == exp_log_message def test_missing_fields(self): @@ -728,12 +731,12 @@ def test_missing_fields(self): # (Spammers can cause this) fv = self.res.forms['dataset-edit'] del fv.fields['log_message'] - res = fv.submit('save', status=400) + res = fv.submit('save', status=400, extra_environ=self.extra_environ_admin) fv = self.res.forms['dataset-edit'] prefix = '' del fv.fields[prefix + 'license_id'] - res = fv.submit('save', status=400) + res = fv.submit('save', status=400, extra_environ=self.extra_environ_admin) def test_redirect_after_edit_using_param(self): return_url = 'http://random.site.com/dataset/?test=param' @@ -741,13 +744,13 @@ def test_redirect_after_edit_using_param(self): # 'http%3A%2F%2Frandom.site.com%2Fdataset%2F%3CNAME%3E%3Ftest%3Dparam' expected_redirect = return_url self._check_redirect(return_url, expected_redirect, - pkg_name_to_edit=self.editpkg_name) + pkg_name_to_edit=self.editpkg_name, extra_environ=self.extra_environ_admin) def test_redirect_after_edit_using_config(self): return_url = '' # redirect comes from test.ini setting expected_redirect = config['package_edit_return_url'] self._check_redirect(return_url, expected_redirect, - pkg_name_to_edit=self.editpkg_name) + pkg_name_to_edit=self.editpkg_name, extra_environ=self.extra_environ_admin) def test_edit_all_fields(self): try: @@ -905,7 +908,7 @@ def test_edit_bad_log_message(self): fv = self.res.forms['dataset-edit'] prefix = '' fv['log_message'] = u'Free enlargements: http://drugs.com/' # spam - res = fv.submit('save') + res = fv.submit('save', extra_environ=self.extra_environ_admin) assert 'Error' in res, res self.check_tag(res, '', + '', '', ] uor_differences = set(uors_str) ^ set(expected_uors_str) From 870f980c35792d0510de649c8db416795edd9008 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 10 Jul 2012 14:14:41 +0100 Subject: [PATCH 024/103] Fixed missing markdown on the resource read page. Changed the read_resource method in package controller to generate the notes from the dictionary and use the dataset notes if no resource description is available --- ckan/controllers/package.py | 5 +++++ ckan/templates/package/resource_read.html | 10 ++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index ccab6931135..dc7ef7708b8 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -772,6 +772,11 @@ def resource_read(self, id, resource_id): c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'), qualified=True) + c.resource_desc_formatted = ckan.misc.MarkdownFormat().to_html( + c.resource.get('description','')) + c.pkg_notes_formatted = ckan.misc.MarkdownFormat().to_html( + c.package.get('notes','')) + c.related_count = len(c.pkg.related) return render('package/resource_read.html') diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index cf03b67f502..7f140947314 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -126,19 +126,21 @@
      -
      +
      - ${c.resource.get('description') or '(No description)'} + ${ Markup(c.resource_desc_formatted) }
      -
      +
      From the Dataset:
      -
      ${h.markdown_extract(c.package.get('notes'), 300)}
      +
      +
      ${ Markup(c.pkg_notes_formatted)}
      +
      From 737d8234ed654fd53e8422205bd23cd824961a0f Mon Sep 17 00:00:00 2001 From: John Glover Date: Tue, 10 Jul 2012 15:18:24 +0100 Subject: [PATCH 025/103] Whitespace cleanup. --- ckan/templates/package/resource_read.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index 7f140947314..6325a0840ab 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -128,7 +128,7 @@
      - ${ Markup(c.resource_desc_formatted) } + ${Markup(c.resource_desc_formatted)}
      @@ -139,7 +139,7 @@
      -
      ${ Markup(c.pkg_notes_formatted)}
      +
      ${Markup(c.pkg_notes_formatted)}
      From f417be9d35a2d3bd17da654a2c9f04f5cbba89d2 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 10 Jul 2012 15:24:41 +0100 Subject: [PATCH 026/103] Add support for TSV files on dataproxy --- ckan/public/scripts/application.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckan/public/scripts/application.js b/ckan/public/scripts/application.js index 131d7b87d8a..7ba52f6ee36 100644 --- a/ckan/public/scripts/application.js +++ b/ckan/public/scripts/application.js @@ -1675,7 +1675,6 @@ CKAN.DataPreview = function ($, my) { // that if we got here (i.e. preview shown) worth doing // something ...) resourceData.formatNormalized = my.normalizeFormat(resourceData.format); - resourceData.url = my.normalizeUrl(resourceData.url); if (resourceData.formatNormalized === '') { var tmp = resourceData.url.split('/'); @@ -1702,7 +1701,7 @@ CKAN.DataPreview = function ($, my) { }); } - else if (resourceData.formatNormalized in {'csv': '', 'xls': ''}) { + else if (resourceData.formatNormalized in {'csv': '', 'xls': '', 'tsv':''}) { // set format as this is used by Recline in setting format for DataProxy resourceData.format = resourceData.formatNormalized; var dataset = new recline.Model.Dataset(resourceData, 'dataproxy'); From fea646fc610c96ae9ff34d38a6dc4f8aa153f4e9 Mon Sep 17 00:00:00 2001 From: Toby Date: Tue, 10 Jul 2012 17:13:34 +0100 Subject: [PATCH 027/103] [#2651] rival patch to keep controller clean --- ckan/controllers/package.py | 5 ----- ckan/lib/helpers.py | 9 +++++++++ ckan/templates/package/resource_read.html | 8 ++++---- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index dc7ef7708b8..ccab6931135 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -772,11 +772,6 @@ def resource_read(self, id, resource_id): c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'), qualified=True) - c.resource_desc_formatted = ckan.misc.MarkdownFormat().to_html( - c.resource.get('description','')) - c.pkg_notes_formatted = ckan.misc.MarkdownFormat().to_html( - c.package.get('notes','')) - c.related_count = len(c.pkg.related) return render('package/resource_read.html') diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 114efb0a78e..a350c1321e4 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -886,6 +886,14 @@ def dashboard_activity_stream(user_id): return logic.get_action('dashboard_activity_list_html')(context, {'id': user_id}) +def render_markdown(data): + ''' returns the data as rendered markdown ''' + # cope with data == None + if not data: + return '' + return literal(ckan.misc.MarkdownFormat().to_html(data)) + + # these are the functions that will end up in `h` template helpers # if config option restrict_template_vars is true __allowed_functions__ = [ @@ -942,6 +950,7 @@ def dashboard_activity_stream(user_id): 'follow_button', 'follow_count', 'dashboard_activity_stream', + 'render_markdown', # imported into ckan.lib.helpers 'literal', 'link_to', diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index 6325a0840ab..7aee8930d46 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -126,20 +126,20 @@
      -
      +
      - ${Markup(c.resource_desc_formatted)} + ${ h.render_markdown(c.resource.get('description')) }
      -
      +
      From the Dataset:
      -
      ${Markup(c.pkg_notes_formatted)}
      +
      ${h.render_markdown(c.package.get('notes')) }
      From f19278e68e8d3148421e97a4a5f034f1b8f94a3b Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Wed, 11 Jul 2012 15:46:12 +0200 Subject: [PATCH 028/103] Stop the package controller from crashing when used at the root URL If the root URL '/' was connected to the package controller (e.g. by an IRoutes plugin) _guess_package_type() would crash. Add a special case to avoid the crash. --- ckan/controllers/package.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index dc7ef7708b8..8113d7d18be 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -93,6 +93,13 @@ def _guess_package_type(self, expecting_name=False): Guess the type of package from the URL handling the case where there is a prefix on the URL (such as /data/package) """ + + # Special case: if the rot URL '/' has been redirected to the package + # controller (e.g. by an IRoutes extension) then there's nothing to do + # here. + if request.path == '/': + return 'dataset' + parts = [x for x in request.path.split('/') if x] idx = -1 From d3056214fbc7e62e0c0e51a907403f86af75491e Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Thu, 12 Jul 2012 10:37:27 +0100 Subject: [PATCH 029/103] [2632] Fix for missing list item with remove --- ckan/config/environment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index f07bb1d979a..9ebcce531d0 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -42,7 +42,8 @@ def __init__(self, helpers, restrict=True): if restrict: continue functions[helper] = getattr(helpers, helper) - allowed.remove(helper) + if helper in allowed: + allowed.remove(helper) self.functions = functions if allowed: From a149644aa8e4933eb69aac9e37d4e8d6c20ed780 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Thu, 12 Jul 2012 11:36:51 +0100 Subject: [PATCH 030/103] [#2313] Revert removal of c.facets. Instead mark it for deprecation in 1.9 --- CHANGELOG.txt | 9 ++++++--- ckan/controllers/group.py | 2 ++ ckan/controllers/home.py | 1 + ckan/controllers/package.py | 2 ++ ckan/logic/action/get.py | 3 +++ 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index ad9c5dd9feb..d280bc1fbea 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -26,9 +26,12 @@ v1.8 API changes and deprecation: * [#2313] Deprecated functions related to the old faceting data structure have - been removed. `helpers.py:facet_items()`, `facets.html:facet_sidebar()`, - `facets.html:facet_list_items()` and the context is no longer populated with - `facets` field, use `search_facets` instead. + been removed: `helpers.py:facet_items()`, `facets.html:facet_sidebar()`, + `facets.html:facet_list_items()`. + Internal use of the old facets datastructure (attached to the context, + `c.facets`) has been superseded by use of the improved facet data structure, + `c.search_facets`. The old data structure is still available on `c.facets`, + but is deprecated, and will be removed in v1.9. v1.7.1 2012-06-20 ================= diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 41bec6ee26f..8bb6119f513 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -200,11 +200,13 @@ def pager_url(q=None, page=None): item_count=query['count'], items_per_page=limit ) + c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Group search error: %r', se.args) c.query_error = True + c.facets = {} c.page = h.Page(collection=[]) # Add the group's activity stream (already rendered to HTML) to the diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index 79e1d1ec889..782f2c8c552 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -49,6 +49,7 @@ def index(self): query = ckan.logic.get_action('package_search')( context, data_dict) c.package_count = query['count'] + c.facets = query['facets'] c.search_facets = query['search_facets'] data_dict = {'order_by': 'packages', 'all_fields': 1} diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 218c746f7db..ccab6931135 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -216,11 +216,13 @@ def pager_url(q=None, page=None): item_count=query['count'], items_per_page=limit ) + c.facets = query['facets'] c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: log.error('Dataset search error: %r', se.args) c.query_error = True + c.facets = {} c.page = h.Page(collection=[]) return render(self._search_template(package_type)) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 460aca6f3b7..27c473e3e96 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1038,6 +1038,8 @@ def package_search(context, data_dict): :param results: ordered list of datasets matching the query, where the ordering defined by the sort parameter used in the query. :type results: list of dictized datasets. + :param facets: DEPRECATED. Aggregated information about facet counts. + :type facets: DEPRECATED dict :param search_facets: aggregated information about facet counts. The outer dict is keyed by the facet field name (as used in the search query). Each entry of the outer dict is itself a dict, with a "title" key, and @@ -1137,6 +1139,7 @@ def package_search(context, data_dict): search_results = { 'count': count, + 'facets': facets, 'results': results } From 8127e208caad9cbbdb3f539cf42502ff5b5af292 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Thu, 12 Jul 2012 11:39:07 +0100 Subject: [PATCH 031/103] [#2313] Remove facet_items from allowed_functions --- ckan/lib/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 0063c9618df..3036df981b4 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -876,7 +876,6 @@ def dashboard_activity_stream(user_id): 'subnav_link', 'subnav_named_route', 'default_group_type', - 'facet_items', 'facet_title', # am_authorized, # deprecated 'check_access', From 6d257fc5298e4e3d5bf56c6e16fb6c9d7ee6eed1 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Thu, 12 Jul 2012 14:10:33 +0100 Subject: [PATCH 032/103] [#2313] Log warning when c.facets is accessed. Using a new `deprecate_context_item` function in `ckan.lib.maintain`, which intercepts access to a named attribute on the context by attaching a property to the underlying storage of the gloal context object. --- ckan/controllers/group.py | 6 ++++++ ckan/controllers/home.py | 6 ++++++ ckan/controllers/package.py | 4 ++++ ckan/lib/maintain.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 8bb6119f513..05179edd3a1 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -7,6 +7,7 @@ from ckan.lib.base import ValidationException, abort, gettext from pylons.i18n import get_lang, _ from ckan.lib.helpers import Page +import ckan.lib.maintain as maintain from ckan.lib.navl.dictization_functions import DataError, unflatten, validate from ckan.logic import NotFound, NotAuthorized, ValidationError from ckan.logic import check_access, get_action @@ -200,7 +201,12 @@ def pager_url(q=None, page=None): item_count=query['count'], items_per_page=limit ) + c.facets = query['facets'] + maintain.deprecate_context_item( + 'facets', + 'Use `c.search_facets` instead.') + c.search_facets = query['search_facets'] c.page.items = query['results'] except SearchError, se: diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index 782f2c8c552..869301d4572 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -4,6 +4,7 @@ import sqlalchemy.exc import ckan.logic +import ckan.lib.maintain as maintain from ckan.lib.search import SearchError from ckan.lib.base import * from ckan.lib.helpers import url_for @@ -49,7 +50,12 @@ def index(self): query = ckan.logic.get_action('package_search')( context, data_dict) c.package_count = query['count'] + c.facets = query['facets'] + maintain.deprecate_context_item( + 'facets', + 'Use `c.search_facets` instead.') + c.search_facets = query['search_facets'] data_dict = {'order_by': 'packages', 'all_fields': 1} diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index ccab6931135..6c65cab3f28 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -15,6 +15,7 @@ model, abort, h, g, c) from ckan.lib.base import response, redirect, gettext +import ckan.lib.maintain as maintain from ckan.lib.package_saver import PackageSaver, ValidationException from ckan.lib.navl.dictization_functions import DataError, unflatten, validate from ckan.lib.helpers import json @@ -225,6 +226,9 @@ def pager_url(q=None, page=None): c.facets = {} c.page = h.Page(collection=[]) + maintain.deprecate_context_item( + 'facets', + 'Use `c.search_facets` instead.') return render(self._search_template(package_type)) def _content_type_from_extension(self, ext): diff --git a/ckan/lib/maintain.py b/ckan/lib/maintain.py index b006c77d3f2..0ded0e72edf 100644 --- a/ckan/lib/maintain.py +++ b/ckan/lib/maintain.py @@ -35,3 +35,36 @@ def wrapped(*args, **kw): return fn(*args, **kw) return wrapped return decorator + +def deprecate_context_item(item_name, message=''): + ''' Deprecate a named item in the global context object. + + It logs a warning when the item is accessed. If a mesage is passed, it is + also logged. This can be useful to indicate for example that a different + function should be used instead. + + No warnings are given when an attempt to change or delete the named item + from the context object. + + Example usage: + + >>> c.facets = "Foobar" + >>> deprecate_context_item('facets', 'Use `c.search_facets` instead') + >>> print c.facets + 2012-07-12 13:27:06,294 WARNI [ckan.lib.maintain] c.facets has been deprecated [...] + Foobar + + This function works by attaching a property to the underlying + `pylons.util.AttribSafeContextObj` object which provides the storage of the + context object. ie - it adds a class-level attribute to the + `pylons.util.AttribSafeContextObj` at runtime. + ''' + # prevent a circular import + from ckan.lib.base import c + + def get_item(self): + log.warning('c.%s has been deprecated. %s', item_name, message) + return getattr(c._current_obj(), item_name) + + setattr(c.__class__, item_name, property(get_item)) + From d051006817f7ff73b51cf0ea797adb9d767da0b5 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 12 Jul 2012 17:06:47 +0100 Subject: [PATCH 033/103] [#1678] Fix check when ckan.plugins line is missing in ini file --- ckan/config/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/config/environment.py b/ckan/config/environment.py index d388937ff33..5272f21031a 100644 --- a/ckan/config/environment.py +++ b/ckan/config/environment.py @@ -129,7 +129,7 @@ def find_controller(self, controller): # Load the synchronous search plugin, unless already loaded or # explicitly disabled - if not 'synchronous_search' in config.get('ckan.plugins') and \ + if not 'synchronous_search' in config.get('ckan.plugins',[]) and \ asbool(config.get('ckan.search.automatic_indexing', True)): log.debug('Loading the synchronous search plugin') p.load('synchronous_search') From 44377670b65b613551ef5219f3aba06cd93e89ba Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Thu, 12 Jul 2012 20:02:17 +0200 Subject: [PATCH 034/103] Fix a typo in a helper function --- ckan/lib/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index b4ff9ee862f..23b6769deca 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -889,7 +889,7 @@ def get_request_param(parameter_name, default=None): ''' This function allows templates to access query string parameters from the request. This is useful for things like sort order in searches. ''' - return request.params.get(parameter_names, default) + return request.params.get(parameter_name, default) def render_markdown(data): ''' returns the data as rendered markdown ''' From b66f54032a0f02fe3f80de9b60b87e3623347785 Mon Sep 17 00:00:00 2001 From: Toby Date: Fri, 13 Jul 2012 13:25:25 +0100 Subject: [PATCH 035/103] [misc] move internal tracking code into a snippet --- ckan/templates/layout_base.html | 24 +-------------- .../templates/snippets/internal-tracking.html | 30 +++++++++++++++++++ 2 files changed, 31 insertions(+), 23 deletions(-) create mode 100644 ckan/templates/snippets/internal-tracking.html diff --git a/ckan/templates/layout_base.html b/ckan/templates/layout_base.html index 4ad1a19a0e9..1ab0ca29a19 100644 --- a/ckan/templates/layout_base.html +++ b/ckan/templates/layout_base.html @@ -265,29 +265,7 @@

      Meta

      - + ${ h.snippet('snippets/internal-tracking.html') }
      diff --git a/ckan/templates/snippets/internal-tracking.html b/ckan/templates/snippets/internal-tracking.html new file mode 100644 index 00000000000..1239850854f --- /dev/null +++ b/ckan/templates/snippets/internal-tracking.html @@ -0,0 +1,30 @@ + + + From d09d24dc519ab792d2bfa3879db4b45ca421ed63 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Fri, 13 Jul 2012 14:14:21 +0100 Subject: [PATCH 036/103] [#2672] Pass in passive=True to Session.is_modified() See [1] for details. [1] http://docs.sqlalchemy.org/en/rel_0_7/orm/session.html?highlight=is_modified#sqlalchemy.orm.session.Session.is_modified --- ckan/lib/dictization/__init__.py | 2 +- ckan/lib/dictization/model_save.py | 2 +- ckan/model/meta.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/lib/dictization/__init__.py b/ckan/lib/dictization/__init__.py index 08999974b7d..6f726ad3436 100644 --- a/ckan/lib/dictization/__init__.py +++ b/ckan/lib/dictization/__init__.py @@ -133,7 +133,7 @@ def table_dict_save(table_dict, ModelClass, context): setattr(obj, key, value) if context.get('pending'): - if session.is_modified(obj, include_collections=False): + if session.is_modified(obj, include_collections=False, passive=True): if table_dict.get('state', '') == 'deleted': obj.state = 'pending-deleted' else: diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index b115aabcd55..64df8723090 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -46,7 +46,7 @@ def resource_dict_save(res_dict, context): del obj.extras[delete_me] if context.get('pending'): - if session.is_modified(obj, include_collections=False): + if session.is_modified(obj, include_collections=False, passive=True): obj.state = u'pending' else: obj.state = u'active' diff --git a/ckan/model/meta.py b/ckan/model/meta.py index df6eef84061..f50285a0e49 100644 --- a/ckan/model/meta.py +++ b/ckan/model/meta.py @@ -60,7 +60,7 @@ def before_flush(self, session, flush_context, instances): 'changed': set()} changed = [obj for obj in session.dirty if - session.is_modified(obj, include_collections=False)] + session.is_modified(obj, include_collections=False, passive=True)] session._object_cache['new'].update(session.new) session._object_cache['deleted'].update(session.deleted) From 62842b71dd4f34145f4057569bae3643ad800a44 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 16 Jul 2012 10:41:01 +0100 Subject: [PATCH 037/103] Update to fix the group.get_children_groups call which was returning deleted items. --- ckan/model/group.py | 5 ++-- ckan/tests/functional/test_group.py | 41 +++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/ckan/model/group.py b/ckan/model/group.py index 415858a5dd2..17156109db6 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -164,7 +164,8 @@ def get_children_groups(self, type='group'): # if returns the full depth of the hierarchy. results = meta.Session.query("id","name", "title").\ from_statement(HIERARCHY_CTE).params(id=self.id, type=type).all() - return [ { "id":idf, "name": name, "title": title } for idf,name,title in results ] + return [{ "id":idf, "name": name, "title": title } + for idf, name, title in results] def active_packages(self, load_eager=True, with_private=False): query = meta.Session.query(_package.Package).\ @@ -287,5 +288,5 @@ def __repr__(self): SELECT G.* FROM subtree AS ST INNER JOIN public.group G ON G.id = ST.table_id - WHERE group_id = :id AND G.type = :type and table_name='group' + WHERE group_id = :id AND G.type = :type and table_name='group' and G.state='active' """ diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index 9ab7c0d2f87..5c95c11335b 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -58,6 +58,47 @@ def setup_class(self): def teardown_class(self): model.repo.rebuild_db() + def test_children(self): + group_name = 'deletetest' + CreateTestData.create_groups([{'name': group_name, + 'packages': []}, + {'name': "parent_group", + 'packages': []}], + admin_user_name='russianfan') + + parent = model.Group.by_name("parent_group") + group = model.Group.by_name(group_name) + + rev = model.repo.new_revision() + rev.author = "none" + + member = model.Member(group_id=parent.id, table_id=group.id, + table_name='group', capacity='member') + model.Session.add(member) + model.repo.commit_and_remove() + + offset = url_for(controller='group', action='edit', id=group_name) + res = self.app.get(offset, status=200, + extra_environ={'REMOTE_USER': 'russianfan'}) + main_res = self.main_div(res) + assert 'Edit: %s' % group.title in main_res, main_res + assert 'value="active" selected' in main_res, main_res + + parent = model.Group.by_name("parent_group") + assert_equal(len(parent.get_children_groups()), 1) + + # delete + form = res.forms['group-edit'] + form['state'] = 'deleted' + res = form.submit('save', status=302, + extra_environ={'REMOTE_USER': 'russianfan'}) + + group = model.Group.by_name(group_name) + assert_equal(group.state, 'deleted') + + parent = model.Group.by_name("parent_group") + assert_equal(len(parent.get_children_groups()), 0) + def test_mainmenu(self): # the home page does a package search so have to skip this test if # search is not supported From 46c514ea18ef74f84ddf71bb0676d77e12059c00 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 16 Jul 2012 10:59:53 +0100 Subject: [PATCH 038/103] PEP8 --- ckan/model/group.py | 183 ++++++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 81 deletions(-) diff --git a/ckan/model/group.py b/ckan/model/group.py index 17156109db6..62767671538 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -15,34 +15,44 @@ 'member_revision_table', 'member_table'] member_table = Table('member', meta.metadata, - Column('id', types.UnicodeText, primary_key=True, default=_types.make_uuid), - Column('table_name', types.UnicodeText, nullable=False), - Column('table_id', types.UnicodeText, nullable=False), - Column('capacity', types.UnicodeText, nullable=False), - Column('group_id', types.UnicodeText, ForeignKey('group.id')), - ) + Column('id', types.UnicodeText, + primary_key=True, + default=_types.make_uuid), + Column('table_name', types.UnicodeText, + nullable=False), + Column('table_id', types.UnicodeText, + nullable=False), + Column('capacity', types.UnicodeText, + nullable=False), + Column('group_id', types.UnicodeText, + ForeignKey('group.id')),) vdm.sqlalchemy.make_table_stateful(member_table) member_revision_table = core.make_revisioned_table(member_table) group_table = Table('group', meta.metadata, - Column('id', types.UnicodeText, primary_key=True, default=_types.make_uuid), - Column('name', types.UnicodeText, nullable=False, unique=True), - Column('title', types.UnicodeText), - Column('type', types.UnicodeText, nullable=False), - Column('description', types.UnicodeText), - Column('image_url', types.UnicodeText), - Column('created', types.DateTime, default=datetime.datetime.now), - Column('approval_status', types.UnicodeText, default=u"approved"), - ) + Column('id', types.UnicodeText, + primary_key=True, + default=_types.make_uuid), + Column('name', types.UnicodeText, + nullable=False, unique=True), + Column('title', types.UnicodeText), + Column('type', types.UnicodeText, + nullable=False), + Column('description', types.UnicodeText), + Column('image_url', types.UnicodeText), + Column('created', types.DateTime, + default=datetime.datetime.now), + Column('approval_status', types.UnicodeText, + default=u"approved")) vdm.sqlalchemy.make_table_stateful(group_table) group_revision_table = core.make_revisioned_table(group_table) class Member(vdm.sqlalchemy.RevisionedObjectMixin, - vdm.sqlalchemy.StatefulObjectMixin, - domain_object.DomainObject): + vdm.sqlalchemy.StatefulObjectMixin, + domain_object.DomainObject): def __init__(self, group=None, table_id=None, group_id=None, table_name=None, capacity='public', state='active'): self.group = group @@ -55,25 +65,27 @@ def __init__(self, group=None, table_id=None, group_id=None, @classmethod def get(cls, reference): '''Returns a group object referenced by its id or name.''' - query = meta.Session.query(cls).filter(cls.id==reference) + query = meta.Session.query(cls).filter(cls.id == reference) member = query.first() - if member == None: + if member is None: member = cls.by_name(reference) return member - def get_related(self, type): """ TODO: Determine if this is useful - Get all objects that are members of the group of the specified type. + Get all objects that are members of the group of the specified + type. - Should the type be used to get table_name or should we use the one in - the constructor + Should the type be used to get table_name or should we use the + one in the constructor """ pass def related_packages(self): # TODO do we want to return all related packages or certain ones? - return meta.Session.query(_package.Package).filter_by(id=self.table_id).all() + return meta.Session.query(_package.Package).filter_by( + id=self.table_id).all() + class Group(vdm.sqlalchemy.RevisionedObjectMixin, vdm.sqlalchemy.StatefulObjectMixin, @@ -86,7 +98,7 @@ def __init__(self, name=u'', title=u'', description=u'', image_url=u'', self.description = description self.image_url = image_url self.type = type - self.approval_status= approval_status + self.approval_status = approval_status @property def display_name(self): @@ -98,9 +110,9 @@ def display_name(self): @classmethod def get(cls, reference): '''Returns a group object referenced by its id or name.''' - query = meta.Session.query(cls).filter(cls.id==reference) + query = meta.Session.query(cls).filter(cls.id == reference) group = query.first() - if group == None: + if group is None: group = cls.by_name(reference) return group # Todo: Make sure group names can't be changed to look like group IDs? @@ -115,7 +127,7 @@ def all(cls, group_type=None, state=('active',)): q = q.filter(cls.state.in_(state)) if group_type: - q = q.filter(cls.type==group_type) + q = q.filter(cls.type == group_type) return q.order_by(cls.title) @@ -135,49 +147,55 @@ def members_of_type(self, object_type, capacity=None): from ckan import model object_type_string = object_type.__name__.lower() query = meta.Session.query(object_type).\ - filter(model.Group.id == self.id).\ - filter(model.Member.state == 'active').\ - filter(model.Member.table_name == object_type_string) + filter(model.Group.id == self.id).\ + filter(model.Member.state == 'active').\ + filter(model.Member.table_name == object_type_string) - if hasattr(object_type,'state'): - query = query.filter(object_type.state == 'active' ) + if hasattr(object_type, 'state'): + query = query.filter(object_type.state == 'active') if capacity: query = query.filter(model.Member.capacity == capacity) - query = query.join(model.Member, member_table.c.table_id == getattr(object_type,'id') ).\ - join(model.Group, group_table.c.id == member_table.c.group_id) + query = query.join(model.Member, member_table.c.table_id == + getattr(object_type, 'id')).\ + join(model.Group, group_table.c.id == member_table.c.group_id) return query def add_child(self, object_instance): object_type_string = object_instance.__class__.__name__.lower() - if not object_instance in self.members_of_type(object_instance.__class__).all(): - member = Member(group=self, table_id=getattr(object_instance,'id'), table_name=object_type_string) + if not object_instance in self.members_of_type( + object_instance.__class__).all(): + member = Member(group=self, + table_id=getattr(object_instance, 'id'), + table_name=object_type_string) meta.Session.add(member) def get_children_groups(self, type='group'): - # Returns a list of dicts where each dict contains "id", "name", and "title" - # When querying with a CTE specifying a model in the query parameter causes - # problems as it returns only the first level deep apparently not recursing - # any deeper than that. If we simplify and request only specific fields then - # if returns the full depth of the hierarchy. - results = meta.Session.query("id","name", "title").\ - from_statement(HIERARCHY_CTE).params(id=self.id, type=type).all() - return [{ "id":idf, "name": name, "title": title } + # Returns a list of dicts where each dict contains "id", "name", + # and "title" When querying with a CTE specifying a model in the + # query parameter causes problems as it returns only the first + # level deep apparently not recursing any deeper than that. If + # we simplify and request only specific fields then if returns + # the full depth of the hierarchy. + results = meta.Session.query("id", "name", "title").\ + from_statement(HIERARCHY_CTE).params(id=self.id, type=type).all() + return [{"id":idf, "name": name, "title": title} for idf, name, title in results] def active_packages(self, load_eager=True, with_private=False): query = meta.Session.query(_package.Package).\ - filter_by(state=vdm.sqlalchemy.State.ACTIVE).\ - filter(group_table.c.id == self.id).\ - filter(member_table.c.state == 'active') + filter_by(state=vdm.sqlalchemy.State.ACTIVE).\ + filter(group_table.c.id == self.id).\ + filter(member_table.c.state == 'active') if not with_private: - query = query.filter(member_table.c.capacity == 'public') + query = query.filter(member_table.c.capacity == 'public') - query = query.join(member_table, member_table.c.table_id == _package.Package.id).\ - join(group_table, group_table.c.id == member_table.c.group_id) + query = query.join(member_table, member_table.c.table_id == + _package.Package.id).\ + join(group_table, group_table.c.id == member_table.c.group_id) return query @@ -186,17 +204,20 @@ def search_by_name_or_title(cls, text_query, group_type=None): text_query = text_query.strip().lower() q = meta.Session.query(cls) \ .filter(or_(cls.name.contains(text_query), - cls.title.ilike('%' + text_query + '%'))) + cls.title.ilike('%' + text_query + '%'))) if group_type: - q = q.filter(cls.type==group_type) + q = q.filter(cls.type == group_type) return q.order_by(cls.title) def as_dict(self, ref_package_by='name'): _dict = domain_object.DomainObject.as_dict(self) - _dict['packages'] = [getattr(package, ref_package_by) for package in self.packages] - _dict['extras'] = dict([(key, value) for key, value in self.extras.items()]) - if ( self.type == 'organization' ): - _dict['users'] = [getattr(user, "name") for user in self.members_of_type(_user.User)] + _dict['packages'] = [getattr(package, ref_package_by) + for package in self.packages] + _dict['extras'] = dict([(key, value) for key, value + in self.extras.items()]) + if (self.type == 'organization'): + _dict['users'] = [getattr(user, "name") + for user in self.members_of_type(_user.User)] return _dict def add_package_by_name(self, package_name): @@ -204,8 +225,9 @@ def add_package_by_name(self, package_name): return package = _package.Package.by_name(package_name) assert package - if not package in self.members_of_type( package.__class__ ).all(): - member = Member(group=self, table_id=package.id, table_name='package') + if not package in self.members_of_type(package.__class__).all(): + member = Member(group=self, table_id=package.id, + table_name='package') meta.Session.add(member) def get_groups(self, group_type=None, capacity=None): @@ -213,10 +235,10 @@ def get_groups(self, group_type=None, capacity=None): import ckan.model as model if '_groups' not in self.__dict__: self._groups = meta.Session.query(model.Group).\ - join(model.Member, model.Member.group_id == model.Group.id and \ - model.Member.table_name == 'group').\ - filter(model.Member.state == 'active').\ - filter(model.Member.table_id == self.id).all() + join(model.Member, model.Member.group_id == model.Group.id and + model.Member.table_name == 'group').\ + filter(model.Member.state == 'active').\ + filter(model.Member.table_id == self.id).all() groups = self._groups if group_type: @@ -225,8 +247,6 @@ def get_groups(self, group_type=None, capacity=None): groups = [g for g in groups if g.capacity == capacity] return groups - - @property def all_related_revisions(self): '''Returns chronological list of all object revisions related to @@ -235,19 +255,20 @@ def all_related_revisions(self): results = {} from group_extra import GroupExtra for grp_rev in self.all_revisions: - if not results.has_key(grp_rev.revision): + if not grp_rev.revision in results: results[grp_rev.revision] = [] results[grp_rev.revision].append(grp_rev) for class_ in [Member, GroupExtra]: rev_class = class_.__revision_class__ - obj_revisions = meta.Session.query(rev_class).filter_by(group_id=self.id).all() + obj_revisions = meta.Session.query(rev_class).\ + filter_by(group_id=self.id).all() for obj_rev in obj_revisions: - if not results.has_key(obj_rev.revision): + if not obj_rev.revision in results: results[obj_rev.revision] = [] results[obj_rev.revision].append(obj_rev) result_list = results.items() ourcmp = lambda rev_tuple1, rev_tuple2: \ - cmp(rev_tuple2[0].timestamp, rev_tuple1[0].timestamp) + cmp(rev_tuple2[0].timestamp, rev_tuple1[0].timestamp) return sorted(result_list, cmp=ourcmp) def __repr__(self): @@ -255,38 +276,38 @@ def __repr__(self): meta.mapper(Group, group_table, - extension=[vdm.sqlalchemy.Revisioner(group_revision_table),], -) + extension=[vdm.sqlalchemy.Revisioner(group_revision_table), ], ) vdm.sqlalchemy.modify_base_object_mapper(Group, core.Revision, core.State) GroupRevision = vdm.sqlalchemy.create_object_version(meta.mapper, Group, - group_revision_table) + group_revision_table) meta.mapper(Member, member_table, properties={ 'group': orm.relation(Group, - backref=orm.backref('member_all', cascade='all, delete-orphan') - ), + backref=orm.backref('member_all', + cascade='all, delete-orphan')), }, - extension=[vdm.sqlalchemy.Revisioner(member_revision_table),], + extension=[vdm.sqlalchemy.Revisioner(member_revision_table), ], ) vdm.sqlalchemy.modify_base_object_mapper(Member, core.Revision, core.State) MemberRevision = vdm.sqlalchemy.create_object_version(meta.mapper, Member, - member_revision_table) + member_revision_table) #TODO MemberRevision.related_packages = lambda self: [self.continuity.package] -HIERARCHY_CTE = """ - WITH RECURSIVE subtree(id) AS ( + +HIERARCHY_CTE = """WITH RECURSIVE subtree(id) AS ( SELECT M.* FROM public.member AS M WHERE M.table_name = 'group' AND M.state = 'active' UNION SELECT M.* FROM public.member M, subtree SG - WHERE M.table_id = SG.group_id AND M.table_name = 'group' AND M.state = 'active') + WHERE M.table_id = SG.group_id AND M.table_name = 'group' + AND M.state = 'active') SELECT G.* FROM subtree AS ST INNER JOIN public.group G ON G.id = ST.table_id - WHERE group_id = :id AND G.type = :type and table_name='group' and G.state='active' -""" + WHERE group_id = :id AND G.type = :type and table_name='group' + and G.state='active'""" From 5739818a528bb16748f67b6d57a16f575ae78144 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Mon, 16 Jul 2012 13:04:33 +0200 Subject: [PATCH 039/103] Fix tracking of dataset views It was counting each view twice, this fixes it to count each view once --- ckan/lib/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 7b9899c6ebf..9029c52442b 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -984,7 +984,7 @@ def update_tracking(self, engine, summary_date): FROM tracking_summary t2 WHERE t1.package_id = t2.package_id AND t2.tracking_date <= t1.tracking_date - ) + t1.count + ) ,recent_views = ( SELECT sum(count) FROM tracking_summary t2 From 0f2d99de74201717bb77d6542c4b7da56aa5a373 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 26 Jun 2012 15:05:49 +0100 Subject: [PATCH 040/103] Don't leave user in limbo if they change their username --- ckan/lib/base.py | 17 +++++++++++------ ckan/templates/user/edit_user_form.html | 7 ++++--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 52604d48e7f..8f4d8a377fd 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -195,12 +195,17 @@ def _identify_user(self): c.userobj = model.User.by_name(c.user) if c.userobj is None: # This occurs when you are logged in, clean db - # and then restart i.e. only really for testers. There is no - # user object, so even though repoze thinks you are logged in - # and your cookie has ckan_display_name, we need to force user - # to logout and login again to get the User object. - c.user = None - self.log.warn('Logout to login') + # and then restart (or when you change your username) + # There is no user object, so even though repoze thinks you + # are logged in and your cookie has ckan_display_name, we + # need to force user to logout and login again to get the + # User object. + session['lang'] = request.environ.get('CKAN_LANG') + session.save() + + pth = getattr(request.environ['repoze.who.plugins']\ + ['friendlyform'], 'logout_handler_path') + h.redirect_to(pth) else: c.userobj = self._get_user_for_apikey() if c.userobj is not None: diff --git a/ckan/templates/user/edit_user_form.html b/ckan/templates/user/edit_user_form.html index 2644a3e7608..7a297d072d6 100644 --- a/ckan/templates/user/edit_user_form.html +++ b/ckan/templates/user/edit_user_form.html @@ -1,6 +1,6 @@ -
      Errors in form
      +

      Changing your username will log you out, and require you to log back in with the new username

      From 0f785050ee39200f090b1ea3e3f00841ab7f79a8 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 26 Jun 2012 15:41:38 +0100 Subject: [PATCH 041/103] PEP8ification --- ckan/lib/base.py | 137 ++++++++++++++++++++++++++--------------------- 1 file changed, 77 insertions(+), 60 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 8f4d8a377fd..dc2cbaaf3d8 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -38,8 +38,9 @@ ALLOWED_FIELDSET_PARAMS = ['package_form', 'restrict'] + def abort(status_code=None, detail='', headers=None, comment=None): - if detail and status_code!=503: + if detail and status_code != 503: h.flash_error(detail) # #1267 Convert detail to plain text, since WebOb 0.9.7.1 (which comes # with Lucid) causes an exception when unicode is received. @@ -59,9 +60,10 @@ def render_snippet(template_name, **kw): cache_force = kw.pop('cache_force', None) output = render(template_name, extra_vars=kw, cache_force=cache_force) output = '\n\n%s\n\n' % ( - template_name, output, template_name) + template_name, output, template_name) return literal(output) + def render_text(template_name, extra_vars=None, cache_force=None): ''' Helper function to render a genshi NewTextTemplate without having to pass the loader_class or method. ''' @@ -71,9 +73,10 @@ def render_text(template_name, extra_vars=None, cache_force=None): method='text', loader_class=NewTextTemplate) + def render(template_name, extra_vars=None, cache_key=None, cache_type=None, cache_expire=None, method='xhtml', loader_class=MarkupTemplate, - cache_force = None): + cache_force=None): ''' Main genshi template rendering function. ''' def render_template(): @@ -89,7 +92,7 @@ def render_template(): del globs['url'] template = globs['app_globals'].genshi_loader.load(template_name, - cls=loader_class) + cls=loader_class) stream = template.generate(**globs) for item in PluginImplementations(IGenshiStreamFilter): @@ -98,8 +101,8 @@ def render_template(): if loader_class == NewTextTemplate: return literal(stream.render(method="text", encoding=None)) - return literal(stream.render(method=method, encoding=None, strip_whitespace=True)) - + return literal(stream.render(method=method, encoding=None, + strip_whitespace=True)) if 'Pragma' in response.headers: del response.headers["Pragma"] @@ -133,7 +136,8 @@ def render_template(): response.headers["Cache-Control"] = "public" try: cache_expire = int(config.get('ckan.cache_expires', 0)) - response.headers["Cache-Control"] += ", max-age=%s, must-revalidate" % cache_expire + response.headers["Cache-Control"] += \ + ", max-age=%s, must-revalidate" % cache_expire except ValueError: pass else: @@ -145,15 +149,18 @@ def render_template(): # Render Time :) try: - return cached_template(template_name, render_template, loader_class=loader_class) + return cached_template(template_name, render_template, + loader_class=loader_class) except ckan.exceptions.CkanUrlException, e: - raise ckan.exceptions.CkanUrlException('\nAn Exception has been raised for template %s\n%s' - % (template_name, e.message)) + raise ckan.exceptions.CkanUrlException( + '\nAn Exception has been raised for template %s\n%s' % + (template_name, e.message)) class ValidationException(Exception): pass + class BaseController(WSGIController): repo = model.repo authorizer = authz.Authorizer() @@ -182,13 +189,16 @@ def _identify_user(self): # see if it was proxied first c.remote_addr = request.environ.get('HTTP_X_FORWARDED_FOR', '') if not c.remote_addr: - c.remote_addr = request.environ.get('REMOTE_ADDR', 'Unknown IP Address') - - # environ['REMOTE_USER'] is set by repoze.who if it authenticates a user's - # cookie or OpenID. But repoze.who doesn't check the user (still) - # exists in our database - we need to do that here. (Another way would - # be with an userid_checker, but that would mean another db access. - # See: http://docs.repoze.org/who/1.0/narr.html#module-repoze.who.plugins.sql ) + c.remote_addr = request.environ.get('REMOTE_ADDR', + 'Unknown IP Address') + + # environ['REMOTE_USER'] is set by repoze.who if it authenticates + # a user's cookie or OpenID. But repoze.who doesn't check the user + # (still) exists in our database - we need to do that here. (Another + # way would be with an userid_checker, but that would mean another db + # access. + # See: http://docs.repoze.org/who/1.0/narr.html#module-repoze.who\ + # .plugins.sql ) c.user = request.environ.get('REMOTE_USER', '') if c.user: c.user = c.user.decode('utf8') @@ -203,8 +213,9 @@ def _identify_user(self): session['lang'] = request.environ.get('CKAN_LANG') session.save() - pth = getattr(request.environ['repoze.who.plugins']\ - ['friendlyform'], 'logout_handler_path') + ev = request.environ + pth = getattr(ev['repoze.who.plugins']['friendlyform'], + 'logout_handler_path') h.redirect_to(pth) else: c.userobj = self._get_user_for_apikey() @@ -246,12 +257,14 @@ def __call__(self, environ, start_response): if not is_valid_cookie_data: if session.id: if not session.get('lang'): - self.log.debug('No session data any more - deleting session') + self.log.debug('No session data any more - ' + 'deleting session') self.log.debug('Session: %r', session.items()) session.delete() else: response.delete_cookie(cookie) - self.log.debug('No session data any more - deleting session cookie') + self.log.debug('No session data any more - ' + 'deleting session cookie') # Remove auth_tkt repoze.who cookie if user not logged in. elif cookie == 'auth_tkt' and not session.id: response.delete_cookie(cookie) @@ -263,8 +276,10 @@ def __after__(self, action, **params): def _set_cors(self): response.headers['Access-Control-Allow-Origin'] = "*" - response.headers['Access-Control-Allow-Methods'] = "POST, PUT, GET, DELETE, OPTIONS" - response.headers['Access-Control-Allow-Headers'] = "X-CKAN-API-KEY, Authorization, Content-Type" + response.headers['Access-Control-Allow-Methods'] = \ + "POST, PUT, GET, DELETE, OPTIONS" + response.headers['Access-Control-Allow-Headers'] = \ + "X-CKAN-API-KEY, Authorization, Content-Type" def _get_user(self, reference): return model.User.by_name(reference) @@ -315,14 +330,14 @@ def _get_request_data(cls, try_url_params=False): keys = request.POST.keys() # Parsing breaks if there is a = in the value, so for now # we will check if the data is actually all in a single key - if keys and request.POST[ keys[0] ] in [u'1',u'']: + if keys and request.POST[keys[0]] in [u'1', u'']: request_data = keys[0] else: request_data = urllib.unquote_plus(request.body) except Exception, inst: msg = "Could not find the POST data: %r : %s" % \ (request.POST, inst) - raise ValueError, msg + raise ValueError(msg) elif try_url_params and request.GET: return request.GET.mixed() @@ -336,22 +351,22 @@ def _get_request_data(cls, try_url_params=False): except Exception, inst: msg = "Could not extract request body data: %s" % \ (inst) - raise ValueError, msg + raise ValueError(msg) cls.log.debug('Retrieved request body: %r' % request.body) if not request_data: msg = "No request body data" - raise ValueError, msg + raise ValueError(msg) if request_data: try: request_data = json.loads(request_data, encoding='utf8') except ValueError, e: - raise ValueError, 'Error decoding JSON data. ' \ - 'Error: %r ' \ - 'JSON data extracted from the request: %r' % \ - (e, request_data) + raise ValueError('Error decoding JSON data. ' + 'Error: %r ' + 'JSON data extracted from the request: %r' % + (e, request_data)) if not isinstance(request_data, dict): - raise ValueError, 'Request data JSON decoded to %r but ' \ - 'it needs to be a dictionary.' % request_data + raise ValueError('Request data JSON decoded to %r but ' + 'it needs to be a dictionary.' % request_data) # ensure unicode values for key, val in request_data.items(): # if val is str then assume it is ascii, since json converts @@ -380,7 +395,8 @@ def _make_unicode(cls, entity): return entity def _get_user_for_apikey(self): - apikey_header_name = config.get(APIKEY_HEADER_NAME_KEY, APIKEY_HEADER_NAME_DEFAULT) + apikey_header_name = config.get(APIKEY_HEADER_NAME_KEY, + APIKEY_HEADER_NAME_DEFAULT) apikey = request.headers.get(apikey_header_name, '') if not apikey: apikey = request.environ.get(apikey_header_name, '') @@ -408,10 +424,10 @@ def _get_timing_cache_path(self): def _get_user_editable_groups(cls): if not hasattr(c, 'user'): c.user = model.PSEUDO_USER__VISITOR - import ckan.authz # Todo: Move import to top of this file? - groups = ckan.authz.Authorizer.authorized_query(c.user, model.Group, - action=model.Action.EDIT).all() - return [g for g in groups if g.state==model.State.ACTIVE] + import ckan.authz # Todo: Move import to top of this file? + groups = ckan.authz.Authorizer.authorized_query( + c.user, model.Group, action=model.Action.EDIT).all() + return [g for g in groups if g.state == model.State.ACTIVE] def _get_package_dict(self, *args, **kwds): import ckan.forms @@ -473,10 +489,10 @@ def _handle_update_of_authz(self, domain_object): update_or_add = None # Work out what role checkboxes are checked or unchecked - checked_roles = [ box_id for (box_id, value) in request.params.items() \ - if (value == u'on')] - unchecked_roles = [ box_id for (box_id, value) in request.params.items() \ - if (value == u'submitted')] + checked_roles = [box_id for (box_id, value) in request.params.items() + if (value == u'on')] + unchecked_roles = [box_id for (box_id, value) in request.params.items() + if (value == u'submitted')] action = None if update_or_add is 'update': @@ -539,42 +555,43 @@ def _prepare_authz_info_for_render(self, user_object_roles): possible_roles = model.Role.get_all() # uniquify and sort - users = sorted(list(set([uor['user_id'] for uor in user_object_roles['roles'] if uor['user_id']]))) - authz_groups = sorted(list(set([uor['authorized_group_id'] \ - for uor in user_object_roles['roles'] \ + users = sorted(list(set([uor['user_id'] + for uor in user_object_roles['roles'] + if uor['user_id']]))) + authz_groups = sorted(list(set([uor['authorized_group_id'] + for uor in user_object_roles['roles'] if uor['authorized_group_id']]))) # make a dictionary from (user, role) to True, False - users_roles = [( uor['user_id'], uor['role']) \ - for uor in user_object_roles['roles'] \ + users_roles = [(uor['user_id'], uor['role']) + for uor in user_object_roles['roles'] if uor['user_id']] - user_role_dict={} + user_role_dict = {} for u in users: for r in possible_roles: - if (u,r) in users_roles: - user_role_dict[(u,r)]=True - else: - user_role_dict[(u,r)]=False + user_role_dict[(u, r)] = (u, r) in users_roles - # and similarly make a dictionary from (authz_group, role) to True, False - authz_groups_roles = [( uor['authorized_group_id'], uor['role']) \ + # and similarly make a dictionary from (authz_group, role) to True + # , False + authz_groups_roles = [(uor['authorized_group_id'], uor['role']) for uor in user_object_roles['roles'] if uor['authorized_group_id']] - authz_groups_role_dict={} + authz_groups_role_dict = {} for u in authz_groups: for r in possible_roles: - if (u,r) in authz_groups_roles: - authz_groups_role_dict[(u,r)]=True + if (u, r) in authz_groups_roles: + authz_groups_role_dict[(u, r)] = True else: - authz_groups_role_dict[(u,r)]=False + authz_groups_role_dict[(u, r)] = False c.roles = possible_roles c.users = users c.user_role_dict = user_role_dict c.authz_groups = authz_groups c.authz_groups_role_dict = authz_groups_role_dict - c.are_any_authz_groups = bool(model.Session.query(model.AuthorizationGroup).count()) + c.are_any_authz_groups = bool(model.Session.query( + model.AuthorizationGroup).count()) # Include the '_' function in the public names -__all__ = [__name for __name in locals().keys() if not __name.startswith('_') \ +__all__ = [__name for __name in locals().keys() if not __name.startswith('_') or __name == '_'] From 0a321150b372ec5620cf8de69d18bb1e6e56a5cb Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 26 Jun 2012 15:42:14 +0100 Subject: [PATCH 042/103] Fix for tests failing in admin due to not loading repoze in test --- ckan/lib/base.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ckan/lib/base.py b/ckan/lib/base.py index dc2cbaaf3d8..b521b3cbeab 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -214,9 +214,10 @@ def _identify_user(self): session.save() ev = request.environ - pth = getattr(ev['repoze.who.plugins']['friendlyform'], - 'logout_handler_path') - h.redirect_to(pth) + if 'repoze.who.plugins' in ev: + pth = getattr(ev['repoze.who.plugins']['friendlyform'], + 'logout_handler_path') + h.redirect_to(pth) else: c.userobj = self._get_user_for_apikey() if c.userobj is not None: From e4fec4ce1dd8eb2abd25c9235a29ff33cb825be6 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 26 Jun 2012 12:37:46 +0100 Subject: [PATCH 043/103] Fix exception when querying and a SolrException occurs, due to not importing it in correct place. --- ckan/lib/search/query.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 25b9ba5bba7..5f1db0fca1a 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -1,4 +1,5 @@ from pylons import config +from solr import SolrException from paste.deploy.converters import asbool from paste.util.multidict import MultiDict from ckan import model @@ -286,7 +287,6 @@ def run(self, query): May raise SearchQueryError or SearchError. ''' - from solr import SolrException assert isinstance(query, (dict, MultiDict)) # check that query keys are valid if not set(query.keys()) <= VALID_SOLR_PARAMETERS: From cd10dc43c4b26637e514aabfc4802f8dd75c5640 Mon Sep 17 00:00:00 2001 From: David Read Date: Wed, 27 Jun 2012 16:10:38 +0100 Subject: [PATCH 044/103] Add cli param to allow options to be passed to celery daemon (e.g. concurrency limit). --- ckan/lib/cli.py | 4 ++- ckan/tests/models/test_user.py | 47 ++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 9029c52442b..681c2ee1c54 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -779,6 +779,7 @@ class Celery(CkanCommand): Usage: celeryd - run the celery daemon celeryd run - run the celery daemon + celeryd run concurrency=1 - run the celery daemon with argument 'concurrency' celeryd view - view all tasks in the queue celeryd clean - delete all tasks in the queue ''' @@ -804,7 +805,8 @@ def command(self): def run_(self): os.environ['CKAN_CONFIG'] = os.path.abspath(self.options.config) from ckan.lib.celery_app import celery - celery.worker_main(argv=['celeryd', '--loglevel=INFO']) + celery_args = ['--%s' % arg for arg in self.args[1:]] + celery.worker_main(argv=['celeryd', '--loglevel=INFO'] + celery_args) def view(self): self._load_config() diff --git a/ckan/tests/models/test_user.py b/ckan/tests/models/test_user.py index a49759a4fc0..ffd3449f13f 100644 --- a/ckan/tests/models/test_user.py +++ b/ckan/tests/models/test_user.py @@ -52,3 +52,50 @@ def test_4_get_openid_missing_slash(self): out = model.User.get(u'http:/sandra.owndomain.com/') assert out assert out.fullname == u'Sandra' + +def to_names(domain_obj_list): + '''Takes a list of domain objects and returns a corresponding list + of their names.''' + objs = [] + for obj in domain_obj_list: + objs.append(obj.name if obj else None) + return objs + +class TestUserGroups: + @classmethod + def setup_class(self): + CreateTestData.create_arbitrary([{'name': 'testpkg'}], + extra_user_names=['brian', 'sandra']) + CreateTestData.create_groups([ + {'name': 'grp1', + 'phone': '1234', + } + ]) + model.repo.new_revision() + grp1 = model.Group.by_name(u'grp1') + brian = model.User.by_name(u'brian') + model.Session.add(model.Member(group=grp1, + table_id=brian.id, + table_name='user', + capacity='admin') + ) + model.repo.commit_and_remove() + + @classmethod + def teardown_class(self): + model.repo.rebuild_db() + + def test_get_groups(self): + brian = model.User.by_name(u'brian') + groups = brian.get_groups() + assert_equal(to_names(groups), ['grp1']) + assert_equal(groups[0].extras, {'phone': '1234'}) + + # check cache works between sessions + model.Session.expunge_all() + #don't refresh brian user since this is how c.user works + # i.e. don't do this: brian = model.User.by_name(u'brian') + groups = brian.get_groups() + assert_equal(to_names(groups), ['grp1']) + assert_equal(groups[0].extras, {'phone': '1234'}) + From 08f14618fa4b06597e3bb6cd1b2746d532e2e6c8 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 10 Jul 2012 11:57:04 +0100 Subject: [PATCH 045/103] Merge conflict --- ckan/controllers/package.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 0b8bf43cd1b..e13a2934daa 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -601,7 +601,11 @@ def _save_new(self, context, package_type=None): except DataError: abort(400, _(u'Integrity Error')) except SearchIndexError, e: - abort(500, _(u'Unable to add package to search index.')) + try: + exc_str = unicode(repr(e.args)) + except: + exc_str = unicode(str(e)) + abort(500, _(u'Unable to add package to search index.') + exc_str) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary @@ -633,7 +637,11 @@ def _save_edit(self, name_or_id, context): except DataError: abort(400, _(u'Integrity Error')) except SearchIndexError, e: - abort(500, _(u'Unable to update search index.')) + try: + exc_str = unicode(repr(e.args)) + except: + exc_str = unicode(str(e)) + abort(500, _(u'Unable to update search index.') + exc_str) except ValidationError, e: errors = e.error_dict error_summary = e.error_summary From c69b08ae548aa957c898db54dd1bd9ed4bfe41e9 Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 10 Jul 2012 11:58:27 +0100 Subject: [PATCH 046/103] Cope with bad revisions being created. To find the problem at source too, but in the mean-time cope. --- ckan/controllers/revision.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ckan/controllers/revision.py b/ckan/controllers/revision.py index e2f9af549c1..798e80995d0 100644 --- a/ckan/controllers/revision.py +++ b/ckan/controllers/revision.py @@ -65,6 +65,11 @@ def list(self): revision_changes[model.ResourceGroup] package_extra_revisions = revision_changes[model.PackageExtra] for package in revision.packages: + if not package: + # package is None sometimes - I don't know why, + # but in the meantime while that is fixed, + # avoid an exception here + continue number = len(package.all_revisions) package_revision = None count = 0 From 4009298f680ece66a5d8c6f784a35e63cc11d1ab Mon Sep 17 00:00:00 2001 From: David Read Date: Tue, 10 Jul 2012 11:58:57 +0100 Subject: [PATCH 047/103] Merge conflicts with cherry-pick --- ckan/lib/dictization/model_save.py | 14 ++++++++++---- ckan/logic/action/create.py | 10 +++++++++- ckan/logic/action/update.py | 22 +++++++++++++++++++--- ckan/new_authz.py | 2 +- 4 files changed, 39 insertions(+), 9 deletions(-) diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index b115aabcd55..a5b3811b33d 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -1,10 +1,13 @@ import datetime import uuid +import logging + from sqlalchemy.orm import class_mapper + import ckan.lib.dictization as d import ckan.lib.helpers as h -##package saving +log = logging.getLogger(__name__) def resource_dict_save(res_dict, context): model = context["model"] @@ -379,9 +382,12 @@ def group_dict_save(group_dict, context): context['group'] = group pkgs_edited = group_member_save(context, group_dict, 'packages') - group_member_save(context, group_dict, 'users') - group_member_save(context, group_dict, 'groups') - group_member_save(context, group_dict, 'tags') + group_users_changed = group_member_save(context, group_dict, 'users') + group_groups_changed = group_member_save(context, group_dict, 'groups') + group_tags_changed = group_member_save(context, group_dict, 'tags') + log.debug('Group save membership changes - Packages: %r Users: %r ' + 'Groups: %r Tags: %r', pkgs_edited, group_users_changed, + group_groups_changed, group_tags_changed) # We will get a list of packages that we have either added or # removed from the group, and trigger a re-index. diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index ce44abfa249..0430a057521 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -127,6 +127,10 @@ def package_create(context, data_dict): package_plugin.check_data_dict(data_dict) data, errors = _validate(data_dict, schema, context) + data, errors = validate(data_dict, schema, context) + log.debug('package_create validate_errs=%r user=%s package=%s data_dict=%r', + errors, context.get('user'), + data_dict.get('name'), data_dict) if errors: model.Session.rollback() @@ -487,7 +491,9 @@ def group_create(context, data_dict): except AttributeError: schema = group_plugin.form_to_db_schema() - data, errors = _validate(data_dict, schema, context) + data, errors = validate(data_dict, schema, context) + log.debug('group_create validate_errs=%r user=%s group=%s data_dict=%r', + errors, context.get('user'), data_dict.get('name'), data_dict) if errors: session.rollback() @@ -508,6 +514,8 @@ def group_create(context, data_dict): if parent_group: member = model.Member(group=parent_group, table_id=group.id, table_name='group') session.add(member) + log.debug('Group %s is made child of group %s', + group.name, parent_group.name) if user: admins = [model.User.by_name(user.decode('utf8'))] diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index ff3c5964043..8d514b1ea89 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -249,7 +249,11 @@ def package_update(context, data_dict): except TypeError: package_plugin.check_data_dict(data_dict) - data, errors = _validate(data_dict, schema, context) + data, errors = validate(data_dict, schema, context) + log.debug('package_update validate_errs=%r user=%s package=%s data_dict=%r', + errors, context.get('user'), + context.get('package').name if context.get('package') else '', + data_dict) if errors: model.Session.rollback() @@ -268,7 +272,9 @@ def package_update(context, data_dict): item.edit(pkg) if not context.get('defer_commit'): model.repo.commit() - return _get_action('package_show')(context, data_dict) + + log.debug('Updated object %s' % str(pkg.name)) + return get_action('package_show')(context, data_dict) def package_update_validate(context, data_dict): model = context['model'] @@ -413,7 +419,12 @@ def group_update(context, data_dict): _check_access('group_update', context, data_dict) - data, errors = _validate(data_dict, schema, context) + data, errors = validate(data_dict, schema, context) + log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r', + errors, context.get('user'), + context.get('group').name if context.get('group') else '', + data_dict) + if errors: session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -435,10 +446,15 @@ def group_update(context, data_dict): current = session.query(model.Member).\ filter(model.Member.table_id == group.id).\ filter(model.Member.table_name == "group").all() + if current: + log.debug('Parents of group %s deleted: %r', group.name, + [membership.group.name for membership in current]) for c in current: session.delete(c) member = model.Member(group=parent_group, table_id=group.id, table_name='group') session.add(member) + log.debug('Group %s is made child of group %s', + group.name, parent_group.name) for item in plugins.PluginImplementations(plugins.IGroupController): diff --git a/ckan/new_authz.py b/ckan/new_authz.py index 9b1a156ef52..ec62d66a171 100644 --- a/ckan/new_authz.py +++ b/ckan/new_authz.py @@ -40,7 +40,7 @@ def _get_auth_function(action, profile=None): if auth_profile: module_root = '%s.%s' % (module_root, auth_profile) - log.info('Using auth profile at %s' % module_root) + log.debug('Using auth profile at %s' % module_root) for auth_module_name in ['get', 'create', 'update','delete']: module_path = '%s.%s' % (module_root, auth_module_name,) From 5644ddd24b2d5d9a170a518a7b9cac2915d419d5 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 12 Jul 2012 12:06:53 +0100 Subject: [PATCH 048/103] Changed debug to show post validation version of the data - more useful. --- ckan/lib/navl/dictization_functions.py | 1 - ckan/logic/action/create.py | 4 ++-- ckan/logic/action/update.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py index aa77d91e530..81676fac868 100644 --- a/ckan/lib/navl/dictization_functions.py +++ b/ckan/lib/navl/dictization_functions.py @@ -302,7 +302,6 @@ def _validate(data, schema, context): except StopOnError: break - return converted_data, errors diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 0430a057521..2b5db0454eb 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -128,9 +128,9 @@ def package_create(context, data_dict): data, errors = _validate(data_dict, schema, context) data, errors = validate(data_dict, schema, context) - log.debug('package_create validate_errs=%r user=%s package=%s data_dict=%r', + log.debug('package_create validate_errs=%r user=%s package=%s data=%r', errors, context.get('user'), - data_dict.get('name'), data_dict) + data.get('name'), data_dict) if errors: model.Session.rollback() diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 8d514b1ea89..535cb57bb15 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -250,10 +250,10 @@ def package_update(context, data_dict): package_plugin.check_data_dict(data_dict) data, errors = validate(data_dict, schema, context) - log.debug('package_update validate_errs=%r user=%s package=%s data_dict=%r', + log.debug('package_update validate_errs=%r user=%s package=%s data=%r', errors, context.get('user'), context.get('package').name if context.get('package') else '', - data_dict) + data) if errors: model.Session.rollback() From 9568c7794957b40021eb8636b47f06c0094b1586 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 13 Jul 2012 10:26:34 +0100 Subject: [PATCH 049/103] Added some useful feedback from these CLI functions. --- ckan/lib/cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 681c2ee1c54..71b8a629765 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -167,12 +167,14 @@ def _postgres_dump(self, filepath): pg_dump_cmd += ' %(db_name)s' % self.db_details pg_dump_cmd += ' > %s' % filepath self._run_cmd(pg_dump_cmd) + print 'Dumped database to: %s' % filepath def _postgres_load(self, filepath): import ckan.model as model assert not model.repo.are_tables_created(), "Tables already found. You need to 'db clean' before a load." pg_cmd = self._get_psql_cmd() + ' -f %s' % filepath self._run_cmd(pg_cmd) + print 'Loaded CKAN database: %s' % filepath def _run_cmd(self, command_line): import subprocess From 3c313968a098413bcf7c92f848693a4149b5280f Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 13 Jul 2012 10:36:40 +0100 Subject: [PATCH 050/103] Merge conflict --- ckan/tests/functional/test_group.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index 5c95c11335b..ec12115d30f 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -836,15 +836,19 @@ def test_edit_success_auth(self): userobj = model.User.get('russianfan') grp = model.Group.by_name(self.groupname) + # Monkey patch + old_method = model.User.get_groups def gg(*args, **kwargs): return [grp] model.User.get_groups = gg - - context = {'group': grp, 'model': model, 'user': 'russianfan'} try: - self.auth.check_access('group_update', context, {}) - except NotAuthorized, e: - assert False, "The user should have access" + context = { 'group': grp, 'model': model, 'user': 'russianfan' } + try: + self.auth.check_access('group_update',context, {}) + except NotAuthorized, e: + assert False, "The user should have access" + finally: + model.User.get_groups = old_method def test_delete(self): group_name = 'deletetest' From 0ec482d2b41e30f91ed136d0ef48c797e746d260 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 13 Jul 2012 10:36:52 +0100 Subject: [PATCH 051/103] [#2668] Sysadmin gets permission to view a deleted publisher. --- ckan/logic/auth/publisher/get.py | 2 ++ ckan/tests/functional/test_publisher_auth.py | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/ckan/logic/auth/publisher/get.py b/ckan/logic/auth/publisher/get.py index 3a76048817d..0e936c95105 100644 --- a/ckan/logic/auth/publisher/get.py +++ b/ckan/logic/auth/publisher/get.py @@ -130,6 +130,8 @@ def group_show(context, data_dict): user = context.get('user') group = get_group_object(context, data_dict) userobj = model.User.get( user ) + if Authorizer().is_sysadmin(unicode(user)): + return {'success': True} if group.state == 'deleted': if not user or \ diff --git a/ckan/tests/functional/test_publisher_auth.py b/ckan/tests/functional/test_publisher_auth.py index 7ea62157656..857c1358224 100644 --- a/ckan/tests/functional/test_publisher_auth.py +++ b/ckan/tests/functional/test_publisher_auth.py @@ -23,8 +23,10 @@ def setup_class(self): model.Session.remove() CreateTestData.create(auth_profile='publisher') self.groupname = u'david' + self.deleted_group_name = u'roger' self.packagename = u'testpkg' model.repo.new_revision() + model.Group.by_name(u'roger').delete() model.Session.add(model.Package(name=self.packagename)) model.repo.commit_and_remove() @@ -34,8 +36,8 @@ def teardown_class(self): model.repo.rebuild_db() model.Session.remove() - def _run_fail_test( self, username, action): - grp = model.Group.by_name(self.groupname) + def _run_fail_test( self, username, action, group_name=None): + grp = model.Group.by_name(group_name or self.groupname) context = { 'group': grp, 'model': model, 'user': username } try: self.auth.check_access(action,context, {}) @@ -43,9 +45,9 @@ def _run_fail_test( self, username, action): except NotAuthorized, e: pass - def _run_success_test( self, username, action): + def _run_success_test( self, username, action, group_name=None): userobj = model.User.get(username) - grp = model.Group.by_name(self.groupname) + grp = model.Group.by_name(group_name or self.groupname) f = model.User.get_groups def gg(*args, **kwargs): return [grp] @@ -100,6 +102,15 @@ def test_delete_anon_fail(self): def test_delete_unknown_fail(self): self._run_fail_test( 'nosuchuser', 'group_delete' ) + def test_read_deleted_success(self): + """ Success because user in group """ + self._run_success_test('testsysadmin', 'group_show', + group_name=self.deleted_group_name) + + def test_read_deleted_fail(self): + """ Fail because user not in group """ + self._run_fail_test('annafan', 'group_show', + group_name=self.deleted_group_name) class TestPublisherShow(FunctionalTestCase): @@ -176,7 +187,6 @@ def gg(*args, **kwargs): model.Package.get_groups = g pkg.state = "active" - class TestPublisherGroupPackages(FunctionalTestCase): From 70dfc9f7fc0a70a525551659034b9eb8959f547a Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Mon, 16 Jul 2012 17:14:30 +0100 Subject: [PATCH 052/103] Fixes from broken merge --- ckan/logic/action/create.py | 5 +---- ckan/logic/action/update.py | 14 +++----------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 2b5db0454eb..469a0c95a02 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -127,7 +127,6 @@ def package_create(context, data_dict): package_plugin.check_data_dict(data_dict) data, errors = _validate(data_dict, schema, context) - data, errors = validate(data_dict, schema, context) log.debug('package_create validate_errs=%r user=%s package=%s data=%r', errors, context.get('user'), data.get('name'), data_dict) @@ -174,7 +173,6 @@ def package_create_validate(context, data_dict): _check_access('package_create',context,data_dict) data, errors = _validate(data_dict, schema, context) - if errors: model.Session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -337,7 +335,6 @@ def package_relationship_create(context, data_dict): return NotFound('Object package %r was not found.' % id2) data, errors = _validate(data_dict, schema, context) - if errors: model.Session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -491,7 +488,7 @@ def group_create(context, data_dict): except AttributeError: schema = group_plugin.form_to_db_schema() - data, errors = validate(data_dict, schema, context) + data, errors = _validate(data_dict, schema, context) log.debug('group_create validate_errs=%r user=%s group=%s data_dict=%r', errors, context.get('user'), data_dict.get('name'), data_dict) diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index 535cb57bb15..69f456a6128 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -137,7 +137,6 @@ def related_update(context, data_dict): _check_access('related_update', context, data_dict) data, errors = _validate(data_dict, schema, context) - if errors: model.Session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -180,7 +179,6 @@ def resource_update(context, data_dict): _check_access('resource_update', context, data_dict) data, errors = _validate(data_dict, schema, context) - if errors: model.Session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -249,7 +247,7 @@ def package_update(context, data_dict): except TypeError: package_plugin.check_data_dict(data_dict) - data, errors = validate(data_dict, schema, context) + data, errors = _validate(data_dict, schema, context) log.debug('package_update validate_errs=%r user=%s package=%s data=%r', errors, context.get('user'), context.get('package').name if context.get('package') else '', @@ -274,7 +272,7 @@ def package_update(context, data_dict): model.repo.commit() log.debug('Updated object %s' % str(pkg.name)) - return get_action('package_show')(context, data_dict) + return _get_action('package_show')(context, data_dict) def package_update_validate(context, data_dict): model = context['model'] @@ -303,8 +301,6 @@ def package_update_validate(context, data_dict): _check_access('package_update', context, data_dict) data, errors = _validate(data_dict, schema, context) - - if errors: model.Session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -365,7 +361,6 @@ def package_relationship_update(context, data_dict): return NotFound('Object package %r was not found.' % id2) data, errors = _validate(data_dict, schema, context) - if errors: model.Session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -419,7 +414,7 @@ def group_update(context, data_dict): _check_access('group_update', context, data_dict) - data, errors = validate(data_dict, schema, context) + data, errors = _validate(data_dict, schema, context) log.debug('group_update validate_errs=%r user=%s group=%s data_dict=%r', errors, context.get('user'), context.get('group').name if context.get('group') else '', @@ -597,7 +592,6 @@ def task_status_update(context, data_dict): _check_access('task_status_update', context, data_dict) data, errors = _validate(data_dict, schema, context) - if errors: session.rollback() raise ValidationError(errors, _error_summary(errors)) @@ -658,7 +652,6 @@ def term_translation_update(context, data_dict): 'lang_code': [validators.not_empty, unicode]} data, errors = _validate(data_dict, schema, context) - if errors: model.Session.rollback() raise ValidationError(errors) @@ -799,7 +792,6 @@ def vocabulary_update(context, data_dict): schema = context.get('schema') or ckan.logic.schema.default_update_vocabulary_schema() data, errors = _validate(data_dict, schema, context) - if errors: model.Session.rollback() raise ValidationError(errors) From a17c5f496a5ae915de8b51b8ff4eeb0847fca947 Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 17 Jul 2012 10:31:31 +0100 Subject: [PATCH 053/103] Fixed bare excepts --- ckan/controllers/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index e13a2934daa..e930f631e9f 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -603,7 +603,7 @@ def _save_new(self, context, package_type=None): except SearchIndexError, e: try: exc_str = unicode(repr(e.args)) - except: + except Exception: # We don't like bare excepts exc_str = unicode(str(e)) abort(500, _(u'Unable to add package to search index.') + exc_str) except ValidationError, e: @@ -639,7 +639,7 @@ def _save_edit(self, name_or_id, context): except SearchIndexError, e: try: exc_str = unicode(repr(e.args)) - except: + except Exception: # We don't like bare excepts exc_str = unicode(str(e)) abort(500, _(u'Unable to update search index.') + exc_str) except ValidationError, e: From 171e20f5a74951c3452941168d3605bc8e1d92cc Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 17 Jul 2012 18:37:50 +0200 Subject: [PATCH 054/103] Fix IGroupForm's setup_template_variables() not being called Call setup_template_variables() methods of IGroupForm plugins when creating new groups. --- ckan/controllers/group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index 8bb6119f513..dd1aba5b50c 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -239,7 +239,7 @@ def new(self, data=None, errors=None, error_summary=None): error_summary = error_summary or {} vars = {'data': data, 'errors': errors, 'error_summary': error_summary} - self._setup_template_variables(context, data) + self._setup_template_variables(context, data, group_type=group_type) c.form = render(self._group_form(group_type=group_type), extra_vars=vars) return render(self._new_template(group_type)) From 2e84da4caa626091863d561d960309de0ac163e1 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 17 Jul 2012 18:41:58 +0200 Subject: [PATCH 055/103] Fix for IGroupForm's form_to_db_schema() not being used when creating datasets The schemas from form_to_db_schema() methods of IGroupForm plugins were not being used when creating groups, fix the code so that they are used. --- ckan/logic/action/create.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index ce44abfa249..8c7690ec3f8 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -479,7 +479,8 @@ def group_create(context, data_dict): _check_access('group_create', context, data_dict) # get the schema - group_plugin = lib_plugins.lookup_group_plugin() + group_plugin = lib_plugins.lookup_group_plugin( + group_type=data_dict.get('type')) try: schema = group_plugin.form_to_db_schema_options({'type':'create', 'api':'api_version' in context, From e9b4f65c4014ce40029f60d643c02e7d6cf832c2 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 17 Jul 2012 18:45:07 +0200 Subject: [PATCH 056/103] Fix data dicts not being validated by group_show group_show() was validating data_dicts but then returning the unvalidated dict not the validated one. Looks like a copy-paste mistake. Fix so that the validated dict is returned. --- ckan/logic/action/get.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index acdab602e75..e1241c110f3 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -691,8 +691,7 @@ def group_show(context, data_dict): schema = group_plugin.db_to_form_schema() if schema: - package_dict, errors = _validate(group_dict, schema, context=context) - + group_dict, errors = _validate(group_dict, schema, context=context) return group_dict def group_package_show(context, data_dict): From 952f28b659aa297cdeaf12d4512da6532cf1c591 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 17 Jul 2012 18:53:55 +0200 Subject: [PATCH 057/103] Fix 500 error on viewing a group with a custom group schema display_name was missing from group_form_schema() so if an IGroupForm plugin with a custom db_to_form_schema() for groups was in use, the group read template would crash with a 500. Add display_name to the schema so that the template renders successfully. --- ckan/logic/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index f3b27654b74..9771740a9f5 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -252,6 +252,7 @@ def group_form_schema(): "capacity": [ignore_missing], "__extras": [ignore] } + schema['display_name'] = [ignore_missing] return schema From 2893c0ca570bd33849dbc5332e5cbf4ebf8f91c2 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 18 Jul 2012 17:04:22 +0100 Subject: [PATCH 058/103] auto summary in validationError --- ckan/logic/__init__.py | 32 +++++++++++++++++++- ckan/templates/package/new_package_form.html | 2 +- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index bb7d94b4b7c..59ceadf71e1 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -1,6 +1,7 @@ import functools import logging import types +import re from ckan.lib.base import _ import ckan.authz @@ -50,9 +51,38 @@ class ParameterError(ActionError): class ValidationError(ParameterError): def __init__(self, error_dict, error_summary=None, extra_msg=None): self.error_dict = error_dict - self.error_summary = error_summary + self._error_summary = error_summary self.extra_msg = extra_msg + @property + def error_summary(self): + if self._error_summary: + print self._error_summary + return self._error_summary + return self._summary(self.error_dict) + + def _summary(self, error_dict): + ''' Do some i18n stuff on the error_dict keys ''' + + def prettify(field_name): + field_name = re.sub('(?Errors in form
    • Resource ${idx}:
        -
      • ${thiskey}: ${errorinfo};
        • +
      • ${thiskey}: ###${errorinfo};
      • From a948489e1c832c72ebed240bcb2cbe8abb420581 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 18 Jul 2012 17:35:27 +0100 Subject: [PATCH 059/103] ValidationError more cleanups --- ckan/logic/__init__.py | 55 ++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 59ceadf71e1..3a5d5fa496f 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -49,39 +49,46 @@ class ParameterError(ActionError): class ValidationError(ParameterError): + def __init__(self, error_dict, error_summary=None, extra_msg=None): + # tags errors are a mess so let's clean them up + error_dict['tags'] = [{'name': [u'Tag ":" length is less than minimum 2', u'Tag ":" must be alphanumeric characters or symbols: -_.']}] + if 'tags' in error_dict: + tag_errors = [] + for error in error_dict['tags']: + tag_errors.append(', '.join(error['name'])) + error_dict['tags'] = tag_errors self.error_dict = error_dict self._error_summary = error_summary self.extra_msg = extra_msg @property def error_summary(self): + ''' autogenerate the summary if not supplied ''' + def summarise(error_dict): + ''' Do some i18n stuff on the error_dict keys ''' + + def prettify(field_name): + field_name = re.sub('(? Date: Wed, 18 Jul 2012 17:39:35 +0100 Subject: [PATCH 060/103] remove idiotic commit --- ckan/templates/package/new_package_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/new_package_form.html b/ckan/templates/package/new_package_form.html index 752ad6ef259..29b3c03797c 100644 --- a/ckan/templates/package/new_package_form.html +++ b/ckan/templates/package/new_package_form.html @@ -20,7 +20,7 @@

      Errors in form

    • Resource ${idx}:
        -
      • ${thiskey}: ###${errorinfo};
        • +
      • ${thiskey}: ${errorinfo};
      • From 76891f4d22073a994301d76ec27e31b224b03aad Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 18 Jul 2012 17:55:05 +0100 Subject: [PATCH 061/103] fix some other silly lines --- ckan/logic/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 3a5d5fa496f..0732c72b5dc 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -52,7 +52,6 @@ class ValidationError(ParameterError): def __init__(self, error_dict, error_summary=None, extra_msg=None): # tags errors are a mess so let's clean them up - error_dict['tags'] = [{'name': [u'Tag ":" length is less than minimum 2', u'Tag ":" must be alphanumeric characters or symbols: -_.']}] if 'tags' in error_dict: tag_errors = [] for error in error_dict['tags']: @@ -86,6 +85,7 @@ def prettify(field_name): else: summary[_(prettify(key))] = error[0] return summary + if self._error_summary: return self._error_summary return summarise(self.error_dict) From 599083ed2b3856b7e9471c7fb62827fd7c9d17c7 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 18 Jul 2012 18:14:26 +0100 Subject: [PATCH 062/103] clean the validation errors in logic action --- ckan/logic/action/create.py | 21 ++++++++++----------- ckan/logic/action/delete.py | 4 ++-- ckan/logic/action/get.py | 20 ++++++++++---------- ckan/logic/action/update.py | 17 ++++++++--------- 4 files changed, 30 insertions(+), 32 deletions(-) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index 8c7690ec3f8..db3efd8819b 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -21,7 +21,6 @@ # Define some shortcuts # Ensure they are module-private so that they don't get loaded as available # actions in the action API. -_error_summary = ckan.logic.action.error_summary _validate = ckan.lib.navl.dictization_functions.validate _check_access = logic.check_access _get_action = logic.get_action @@ -130,7 +129,7 @@ def package_create(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user @@ -173,7 +172,7 @@ def package_create_validate(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) else: return data @@ -237,7 +236,7 @@ def resource_create(context, data_dict): pkg_dict = _get_action('package_update')(context, pkg_dict) except ValidationError, e: errors = e.error_dict['resources'][-1] - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) return pkg_dict['resources'][-1] @@ -278,7 +277,7 @@ def related_create(context, data_dict): context) if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) related = model_save.related_dict_save(data, context) if not context.get('defer_commit'): @@ -336,7 +335,7 @@ def package_relationship_create(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) _check_access('package_relationship_create', context, data_dict) @@ -492,7 +491,7 @@ def group_create(context, data_dict): if errors: session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user @@ -631,7 +630,7 @@ def user_create(context, data_dict): if errors: session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) user = model_save.user_dict_save(data, context) @@ -720,7 +719,7 @@ def vocabulary_create(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) vocabulary = model_save.vocabulary_dict_save(data, context) @@ -858,7 +857,7 @@ def follow_user(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) # Don't let a user follow herself. if userobj.id == data_dict['id']: @@ -930,7 +929,7 @@ def follow_dataset(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) # Don't let a user follow a dataset she is already following. if model.UserFollowingDataset.get(userobj.id, data_dict['id']) is not None: diff --git a/ckan/logic/action/delete.py b/ckan/logic/action/delete.py index c2d37519351..6e7ad7d3609 100644 --- a/ckan/logic/action/delete.py +++ b/ckan/logic/action/delete.py @@ -297,7 +297,7 @@ def unfollow_user(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) _unfollow(context, data_dict, context['model'].UserFollowingUser) @@ -312,6 +312,6 @@ def unfollow_dataset(context, data_dict): ckan.logic.schema.default_follow_dataset_schema()) data_dict, errors = validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) _unfollow(context, data_dict, context['model'].UserFollowingDataset) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index e1241c110f3..3d1090f3cf5 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1972,7 +1972,7 @@ def user_follower_count(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return ckan.model.UserFollowingUser.follower_count(data_dict['id']) def dataset_follower_count(context, data_dict): @@ -1988,7 +1988,7 @@ def dataset_follower_count(context, data_dict): ckan.logic.schema.default_follow_dataset_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return ckan.model.UserFollowingDataset.follower_count(data_dict['id']) def _follower_list(context, data_dict, FollowerClass): @@ -2017,7 +2017,7 @@ def user_follower_list(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return _follower_list(context, data_dict, context['model'].UserFollowingUser) @@ -2034,7 +2034,7 @@ def dataset_follower_list(context, data_dict): ckan.logic.schema.default_follow_dataset_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return _follower_list(context, data_dict, context['model'].UserFollowingDataset) @@ -2065,7 +2065,7 @@ def am_following_user(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return _am_following(context, data_dict, context['model'].UserFollowingUser) @@ -2083,7 +2083,7 @@ def am_following_dataset(context, data_dict): ckan.logic.schema.default_follow_dataset_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return _am_following(context, data_dict, context['model'].UserFollowingDataset) @@ -2101,7 +2101,7 @@ def user_followee_count(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return ckan.model.UserFollowingUser.followee_count(data_dict['id']) def dataset_followee_count(context, data_dict): @@ -2117,7 +2117,7 @@ def dataset_followee_count(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) return ckan.model.UserFollowingDataset.followee_count(data_dict['id']) def user_followee_list(context, data_dict): @@ -2133,7 +2133,7 @@ def user_followee_list(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) # Get the list of Follower objects. model = context['model'] @@ -2160,7 +2160,7 @@ def dataset_followee_list(context, data_dict): ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) if errors: - raise ValidationError(errors, ckan.logic.action.error_summary(errors)) + raise ValidationError(errors) # Get the list of Follower objects. model = context['model'] diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py index ff3c5964043..3dd79e5a3f3 100644 --- a/ckan/logic/action/update.py +++ b/ckan/logic/action/update.py @@ -20,7 +20,6 @@ # Ensure they are module-private so that they don't get loaded as available # actions in the action API. _validate = ckan.lib.navl.dictization_functions.validate -_error_summary = logic.action.error_summary _get_action = logic.get_action _check_access = logic.check_access NotFound = logic.NotFound @@ -140,7 +139,7 @@ def related_update(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) related = model_save.related_dict_save(data, context) if not context.get('defer_commit'): @@ -183,7 +182,7 @@ def resource_update(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user @@ -253,7 +252,7 @@ def package_update(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user @@ -301,7 +300,7 @@ def package_update_validate(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) return data @@ -362,7 +361,7 @@ def package_relationship_update(context, data_dict): if errors: model.Session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) _check_access('package_relationship_update', context, data_dict) @@ -416,7 +415,7 @@ def group_update(context, data_dict): data, errors = _validate(data_dict, schema, context) if errors: session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) rev = model.repo.new_revision() rev.author = user @@ -514,7 +513,7 @@ def user_update(context, data_dict): data, errors = _validate(data_dict, schema, context) if errors: session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) user = model_save.user_dict_save(data, context) @@ -584,7 +583,7 @@ def task_status_update(context, data_dict): if errors: session.rollback() - raise ValidationError(errors, _error_summary(errors)) + raise ValidationError(errors) task_status = model_save.task_status_dict_save(data, context) From e7db091942c085d51765dbbdf4a85b785df7174d Mon Sep 17 00:00:00 2001 From: tobes Date: Fri, 20 Jul 2012 13:38:48 +0100 Subject: [PATCH 063/103] fix for failing tests --- ckan/logic/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index 0732c72b5dc..49a1a4c8828 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -55,7 +55,10 @@ def __init__(self, error_dict, error_summary=None, extra_msg=None): if 'tags' in error_dict: tag_errors = [] for error in error_dict['tags']: - tag_errors.append(', '.join(error['name'])) + try: + tag_errors.append(', '.join(error['name'])) + except KeyError: + pass error_dict['tags'] = tag_errors self.error_dict = error_dict self._error_summary = error_summary From 7c8d45062b60d72bdbc3522341ea109abdbcee39 Mon Sep 17 00:00:00 2001 From: tobes Date: Fri, 20 Jul 2012 16:06:05 +0100 Subject: [PATCH 064/103] clean up the concurrency option too stop passing other args --- ckan/lib/cli.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 71b8a629765..eabc963e503 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -781,11 +781,12 @@ class Celery(CkanCommand): Usage: celeryd - run the celery daemon celeryd run - run the celery daemon - celeryd run concurrency=1 - run the celery daemon with argument 'concurrency' + celeryd run concurrency - run the celery daemon with argument 'concurrency' celeryd view - view all tasks in the queue celeryd clean - delete all tasks in the queue ''' min_args = 0 + max_args = 2 summary = __doc__.split('\n')[0] usage = __doc__ @@ -807,7 +808,9 @@ def command(self): def run_(self): os.environ['CKAN_CONFIG'] = os.path.abspath(self.options.config) from ckan.lib.celery_app import celery - celery_args = ['--%s' % arg for arg in self.args[1:]] + celery_args = [] + if len(self.args) == 2 and self.args[1] == 'concurrency': + celery_args.append['--concurrency=1'] celery.worker_main(argv=['celeryd', '--loglevel=INFO'] + celery_args) def view(self): From 7e28eed0fdb951645454990b794dac1b117b1403 Mon Sep 17 00:00:00 2001 From: tobes Date: Fri, 20 Jul 2012 16:06:24 +0100 Subject: [PATCH 065/103] remove unused import --- ckan/lib/cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index eabc963e503..f77affbad73 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -830,7 +830,6 @@ def view(self): def clean(self): self._load_config() import ckan.model as model - import pprint tasks_initially = model.Session.execute("select * from kombu_message").rowcount if not tasks_initially: print 'No tasks to delete' From 46b652c69464c932cac6f677a4ba5d3775bd99c9 Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 23 Jul 2012 09:19:00 +0100 Subject: [PATCH 066/103] pep8 fixes for from dgu branch cli changes --- ckan/lib/cli.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index f77affbad73..3aef7efe8e8 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -779,11 +779,12 @@ class Celery(CkanCommand): '''Celery daemon Usage: - celeryd - run the celery daemon - celeryd run - run the celery daemon - celeryd run concurrency - run the celery daemon with argument 'concurrency' - celeryd view - view all tasks in the queue - celeryd clean - delete all tasks in the queue + celeryd - run the celery daemon + celeryd run - run the celery daemon + celeryd run concurrency - run the celery daemon with + argument 'concurrency' + celeryd view - view all tasks in the queue + celeryd clean - delete all tasks in the queue ''' min_args = 0 max_args = 2 @@ -830,12 +831,14 @@ def view(self): def clean(self): self._load_config() import ckan.model as model - tasks_initially = model.Session.execute("select * from kombu_message").rowcount + query = model.Session.execute("select * from kombu_message") + tasks_initially = query.rowcount if not tasks_initially: print 'No tasks to delete' sys.exit(0) query = model.Session.execute("delete from kombu_message") - tasks_afterwards = model.Session.execute("select * from kombu_message").rowcount + query = model.Session.execute("select * from kombu_message") + tasks_afterwards = query.rowcount print '%i of %i tasks deleted' % (tasks_initially - tasks_afterwards, tasks_initially) if tasks_afterwards: @@ -843,6 +846,7 @@ def clean(self): sys.exit(1) model.repo.commit_and_remove() + class Ratings(CkanCommand): '''Manage the ratings stored in the db From 3ee86db592d5717ac426b46b91f16c4bc3efc51a Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 23 Jul 2012 14:43:49 +0100 Subject: [PATCH 067/103] [#2575] fix license title translation --- ckan/model/license.py | 404 +++++++++++++++++++++--------------------- 1 file changed, 203 insertions(+), 201 deletions(-) diff --git a/ckan/model/license.py b/ckan/model/license.py index c31fb218ebf..2766d97c38d 100644 --- a/ckan/model/license.py +++ b/ckan/model/license.py @@ -49,207 +49,21 @@ def __init__(self): self.load_licenses(group_url) else: default_license_list = [ - { - "domain_content": False, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "notspecified", - "is_generic": True, - "is_okd_compliant": False, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": _("License Not Specified"), - "url": "" - }, - { - "domain_content": False, - "domain_data": True, - "domain_software": False, - "family": "", - "id": "odc-pddl", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "Open Data Commons Public Domain Dedication and Licence (PDDL)", - "url": "http://www.opendefinition.org/licenses/odc-pddl" - }, - { - "domain_content": False, - "domain_data": True, - "domain_software": False, - "family": "", - "id": "odc-odbl", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "Open Data Commons Open Database License (ODbL)", - "url": "http://www.opendefinition.org/licenses/odc-odbl" - }, - { - "domain_content": False, - "domain_data": True, - "domain_software": False, - "family": "", - "id": "odc-by", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "Open Data Commons Attribution License", - "url": "http://www.opendefinition.org/licenses/odc-by" - }, - { - "domain_content": True, - "domain_data": True, - "domain_software": False, - "family": "", - "id": "cc-zero", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "Creative Commons CCZero", - "url": "http://www.opendefinition.org/licenses/cc-zero" - }, - { - "domain_content": True, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "cc-by", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "Creative Commons Attribution", - "url": "http://www.opendefinition.org/licenses/cc-by" - }, - { - "domain_content": True, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "cc-by-sa", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "Creative Commons Attribution Share-Alike", - "url": "http://www.opendefinition.org/licenses/cc-by-sa" - }, - { - "domain_content": True, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "gfdl", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "GNU Free Documentation License", - "url": "http://www.opendefinition.org/licenses/gfdl" - }, - { - "domain_content": True, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "other-open", - "is_generic": True, - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": _("Other (Open)"), - "url": "" - }, - { - "domain_content": True, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "other-pd", - "is_generic": True, - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": _("Other (Public Domain)"), - "url": "" - }, - { - "domain_content": True, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "other-at", - "is_generic": True, - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": _("Other (Attribution)"), - "url": "" - }, - { - "domain_content": True, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "uk-ogl", - "is_okd_compliant": True, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "UK Open Government Licence (OGL)", - "url": "http://reference.data.gov.uk/id/open-government-licence" - }, - { - "domain_content": False, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "cc-nc", - "is_okd_compliant": False, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": "Creative Commons Non-Commercial (Any)", - "url": "http://creativecommons.org/licenses/by-nc/2.0/" - }, - { - "domain_content": False, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "other-nc", - "is_generic": True, - "is_okd_compliant": False, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": _("Other (Non-Commercial)"), - "url": "" - }, - { - "domain_content": False, - "domain_data": False, - "domain_software": False, - "family": "", - "id": "other-closed", - "is_generic": True, - "is_okd_compliant": False, - "is_osi_compliant": False, - "maintainer": "", - "status": "active", - "title": _("Other (Not Open)"), - "url": "" - } + LicenseNotSpecified(), + LicenseOpenDataCommonsPDDL(), + LicenseOpenDataCommonsOpenDatabase(), + LicenseOpenDataAttribution(), + LicenseCreativeCommonsZero(), + LicenseCreativeCommonsAttribution(), + LicenseCreativeCommonsAttributionShareAlike(), + LicenseGNUFreeDocument(), + LicenseOtherOpen(), + LicenseOtherPublicDomain(), + LicenseOtherAttribution(), + LicenseOpenGovernment(), + LicenseCreativeCommonsNonCommercial(), + LicenseOtherNonCommercial(), + LicenseOtherClosed(), ] self._create_license_list(default_license_list) @@ -302,3 +116,191 @@ def __iter__(self): def __len__(self): return len(self.licenses) + + + +class DefaultLicense(dict): + ''' The license was a dict but this did not allow translation of the + title. This is a slightly changed dict that allows us to have the title + as a property and so translated. ''' + + domain_content = False + domain_data = False + domain_software = False + family = "" + is_generic = False + is_okd_compliant = False + is_osi_compliant = False + maintainer = "" + status = "active" + url = "" + title = '' + id = '' + + keys = ['domain_content', + 'id', + 'domain_data', + 'domain_software', + 'family', + 'is_generic', + 'is_okd_compliant', + 'is_osi_compliant', + 'maintainer', + 'status', + 'url', + 'title'] + + def __getitem__(self, key): + ''' behave like a dict but get from attributes ''' + if key in self.keys: + return unicode(getattr(self, key)) + else: + raise KeyError() + + def copy(self): + ''' create a dict of the license used by the licenses api ''' + out = {} + for key in self.keys: + out[key] = unicode(getattr(self, key)) + return out + +class LicenseNotSpecified(DefaultLicense): + id = "notspecified" + is_generic = True + + @property + def title(self): + return _("License Not Specified") + +class LicenseOpenDataCommonsPDDL(DefaultLicense): + domain_data = True + id = "odc-pddl" + is_okd_compliant = True + url = "http://www.opendefinition.org/licenses/odc-pddl" + + @property + def title(self): + return _("Open Data Commons Public Domain Dedication and Licence (PDDL)") + +class LicenseOpenDataCommonsOpenDatabase(DefaultLicense): + domain_data = True + id = "odc-odbl" + is_okd_compliant = True + url = "http://www.opendefinition.org/licenses/odc-odbl" + + @property + def title(self): + return _("Open Data Commons Open Database License (ODbL)") + +class LicenseOpenDataAttribution(DefaultLicense): + domain_data = True + id = "odc-by" + is_okd_compliant = True + url = "http://www.opendefinition.org/licenses/odc-by" + + @property + def title(self): + return _("Open Data Commons Attribution License") + +class LicenseCreativeCommonsZero(DefaultLicense): + domain_content = True + domain_data = True + id = "cc-zero" + is_okd_compliant = True + url = "http://www.opendefinition.org/licenses/cc-zero" + + @property + def title(self): + return _("Creative Commons CCZero") + +class LicenseCreativeCommonsAttribution(DefaultLicense): + id = "cc-by" + is_okd_compliant = True + url = "http://www.opendefinition.org/licenses/cc-by" + + @property + def title(self): + return _("Creative Commons Attribution") + +class LicenseCreativeCommonsAttributionShareAlike(DefaultLicense): + domain_content = True + id = "cc-by-sa" + is_okd_compliant = True + url = "http://www.opendefinition.org/licenses/cc-by-sa" + + @property + def title(self): + return _("Creative Commons Attribution Share-Alike") + +class LicenseGNUFreeDocument(DefaultLicense): + domain_content = True + id = "gfdl" + is_okd_compliant = True + url = "http://www.opendefinition.org/licenses/gfdl" + @property + def title(self): + return _("GNU Free Documentation License") + +class LicenseOtherOpen(DefaultLicense): + domain_content = True + id = "other-open" + is_generic = True + is_okd_compliant = True + + @property + def title(self): + return _("Other (Open)") + +class LicenseOtherPublicDomain(DefaultLicense): + domain_content = True + id = "other-pd" + is_generic = True + is_okd_compliant = True + + @property + def title(self): + return _("Other (Public Domain)") + +class LicenseOtherAttribution(DefaultLicense): + domain_content = True + id = "other-at" + is_generic = True + is_okd_compliant = True + + @property + def title(self): + return _("Other (Attribution)") + +class LicenseOpenGovernment(DefaultLicense): + domain_content = True + id = "uk-ogl" + is_okd_compliant = True + url = "http://reference.data.gov.uk/id/open-government-licence" + + @property + def title(self): + return _("UK Open Government Licence (OGL)") + +class LicenseCreativeCommonsNonCommercial(DefaultLicense): + id = "cc-nc" + url = "http://creativecommons.org/licenses/by-nc/2.0/" + + @property + def title(self): + return _("Creative Commons Non-Commercial (Any)") + +class LicenseOtherNonCommercial(DefaultLicense): + id = "other-nc" + is_generic = True + + @property + def title(self): + return _("Other (Non-Commercial)") + +class LicenseOtherClosed(DefaultLicense): + id = "other-closed" + is_generic = True + + @property + def title(self): + return _("Other (Not Open)") From 9fb1d16e7bf8465f205d35ff648ca207ae9d4675 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 24 Jul 2012 10:52:50 +0100 Subject: [PATCH 068/103] [#2375] start of admin template fixes --- ckan/templates/admin/authz.html | 161 ++++++++++++++++++++++++++++++++ ckan/templates/admin/index.html | 26 ++++++ 2 files changed, 187 insertions(+) create mode 100644 ckan/templates/admin/authz.html create mode 100644 ckan/templates/admin/index.html diff --git a/ckan/templates/admin/authz.html b/ckan/templates/admin/authz.html new file mode 100644 index 00000000000..1491d6ad92d --- /dev/null +++ b/ckan/templates/admin/authz.html @@ -0,0 +1,161 @@ +{% extends "page.html" %} + +{% block title %}{{ _('Administration - Authorization') }} - {{ super() }}{% endblock %} + +{% block breadcrumb_content %} +{% endblock %} + +{% macro authz_form_table(id, roles, users, user_role_dict) %} + + + + {% for role in roles %} + + {% endfor %} + + {% for user in users %} + + + {% for role in roles %} + + {% endfor %} + + {% endfor %} +
      User {{ role }}
      + {{ h.linked_user(user) }} + + + {% if user_role_dict[(user,role)] %} + + {% else %} + + {% endif %} +
      +{% endmacro %} + + +{% macro authz_add_table(roles) %} + + + + {% for role in roles %} + + {% endfor %} + + + + {% for role in roles %} + + {% endfor %} + +
      User {{ role }}
      + + + + +
      +{% endmacro %} + +{% macro authz_form_group_table(id, roles, users, user_role_dict) %} + + + + {% for role in roles %} + + {% endfor %} + + {% for user in users %} + + + {% for role in roles %} + + {% endfor %} + + {% endfor %} +
      User Group {{ role }}
      + {{ h.linked_authorization_group(user) }} + + + {% if user_role_dict[(user,role)] %} + + {% else %} + + {% endif %} +
      +{% endmacro %} + + +{% macro authz_add_group_table(roles) %} + + + + {% for role in roles %} + + {% endfor %} + + + + {% for role in roles %} + + {% endfor %} + +
      User Group {{ role }}
      + + + + +
      +{% endmacro %} + +{% block primary %} +
      +
      +

      {{ _('Administration - Authorization') }}

      +

      Update Existing Roles

      + + {{ authz_form_table('theform', c.roles, c.users, c.user_role_dict) }} + +
      + + +

      Add Roles for Any User

      +
      + {{ authz_add_table(c.roles) }} + +
      +
      + +
      + + {% if c.are_any_authz_groups %} +

      Existing Roles for Authorization Groups

      + +
      + {{ authz_form_group_table('authzgroup_form', c.roles, c.authz_groups, c.authz_groups_role_dict) }} + +
      +
      + +

      Add Roles for Any Authorization Group

      + +
      + {{ authz_add_group_table(c.roles) }} + +
      +
      + {% endif %} +
      +
      +{% endblock %} + +{% block sidebar %}{% endblock %} diff --git a/ckan/templates/admin/index.html b/ckan/templates/admin/index.html new file mode 100644 index 00000000000..10b3ef863fd --- /dev/null +++ b/ckan/templates/admin/index.html @@ -0,0 +1,26 @@ +{% extends "page.html" %} + +{% block title %}{{ _('Administration') }} - {{ super() }}{% endblock %} + +{% block breadcrumb_content %} +{% endblock %} + +{% block primary %} +
      +
      +

      {{ _('Current Sysadmins') }}

      +

      You can change sysadmins on the + authorization page + .

      +
        + {% for user in c.sysadmins %} +
      • + {{ h.linked_user(user) }} +
        • + {% endfor %} +
      +
      +
      +{% endblock %} + +{% block sidebar %}{% endblock %} From b3c1f2b633813394fb2ff2c92af0b5b0ce82d642 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 24 Jul 2012 11:33:37 +0100 Subject: [PATCH 069/103] [#2734] resource edit error fixed --- ckan/templates/package/snippets/resource_form.html | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ckan/templates/package/snippets/resource_form.html b/ckan/templates/package/snippets/resource_form.html index 153d42cdfce..5ddbfbf4961 100644 --- a/ckan/templates/package/snippets/resource_form.html +++ b/ckan/templates/package/snippets/resource_form.html @@ -56,12 +56,6 @@ {{ form.input('mimetype', id='field-mimetype', label=_('MIME Type'), placeholder=_('eg. application/json'), value=data.mimetype, error=errors.mimetype, classes=[]) }} {{ form.input('mimetype_inner', id='field-mimetype-inner', label=_('MIME Type'), placeholder=_('eg. application/json'), value=data.mimetype_inner, error=errors.mimetype_inner, classes=[]) }} - {% else %} - - - - - {% endif %} {% endblock %} From b255e12cbc68037d4992656dae1da5b528acf261 Mon Sep 17 00:00:00 2001 From: tobes Date: Mon, 23 Jul 2012 17:19:37 +0100 Subject: [PATCH 070/103] Treat commas in tags as returns This allows the user to create a new tag when pressing comma. --- .../base/javascript/modules/autocomplete.js | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js index 907df4a9fbd..64bcc231d89 100644 --- a/ckan/public/base/javascript/modules/autocomplete.js +++ b/ckan/public/base/javascript/modules/autocomplete.js @@ -56,15 +56,16 @@ this.ckan.module('autocomplete', function (jQuery, _) { // tags or generic completion. if (this.options.tags) { settings.tags = this._onQuery; - - // Also need to watch for changes so we can handle formatting - // inconsistencies that occur when dealing with tags. - this.el.on('change', this._onChange); } else { settings.query = this._onQuery; } this.el.select2(settings); + + if (this.options.tags) { + // find the "fake" input created by select2 and add the keypress event + this.el.next().find('input').on('keydown', this._onKeydown); + } }, /* Looks up the completions for the current search term and passes them @@ -195,19 +196,19 @@ this.ckan.module('autocomplete', function (jQuery, _) { this.lookup(options.term, options.callback); }, - /* Called when the input changes. Used to split any comma separated tags - * into individual items. This is a bit of a workaround as select2 doesn't - * handle this yet. - * - * select2('val') actually parses comma separated input correctly but - * doesn't render them. So we give it a gentle nudge. + /* Called when a key is pressed. If the key is a comma we block it and + * then simulate pressing return. * * Returns nothing. */ - _onChange: function (event) { - var parsed = jQuery.map(this.el.select2('val'), this.formatTerm); - - this.el.select2('val', parsed); + _onKeydown: function (event) { + if (event.which === 188) { + event.preventDefault(); + setTimeout(function () { + var e = jQuery.Event("keydown", { which: 13 }); + jQuery(event.target).trigger(e); + }, 10); + } } }; }); From 268473c4c648eb61eb9c1409176b90c5de6ba205 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 11:35:13 +0100 Subject: [PATCH 071/103] [#2695] Add a jshint.json with project specific settings --- jshint.json | 1 + 1 file changed, 1 insertion(+) create mode 100644 jshint.json diff --git a/jshint.json b/jshint.json new file mode 100644 index 00000000000..38a5cb5079c --- /dev/null +++ b/jshint.json @@ -0,0 +1 @@ +{"onevar": false} From d353d53a843261be77880e7d15ca088fbef2da4e Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 11:39:47 +0100 Subject: [PATCH 072/103] [#2695] Improve reliability of autocomplete module --- ckan/public/base/javascript/modules/autocomplete.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js index 64bcc231d89..c85d9082f12 100644 --- a/ckan/public/base/javascript/modules/autocomplete.js +++ b/ckan/public/base/javascript/modules/autocomplete.js @@ -60,11 +60,12 @@ this.ckan.module('autocomplete', function (jQuery, _) { settings.query = this._onQuery; } - this.el.select2(settings); + var select2 = this.el.select2(settings).data('select2'); - if (this.options.tags) { - // find the "fake" input created by select2 and add the keypress event - this.el.next().find('input').on('keydown', this._onKeydown); + if (this.options.tags && select2 && select2.search) { + // find the "fake" input created by select2 and add the keypress event. + // This is not part of the plugins API and so may break at any time. + select2.search.on('keydown', this._onKeydown); } }, From 0161238b9b446b8424bd3f3daa5394bdd963b697 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 12:00:49 +0100 Subject: [PATCH 073/103] [#2695] Update unit tests for autocomplete _onKeydown() handler --- .../test/spec/modules/autocomplete.spec.js | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/ckan/public/base/test/spec/modules/autocomplete.spec.js b/ckan/public/base/test/spec/modules/autocomplete.spec.js index f289dc0e2a0..96fec673b82 100644 --- a/ckan/public/base/test/spec/modules/autocomplete.spec.js +++ b/ckan/public/base/test/spec/modules/autocomplete.spec.js @@ -7,7 +7,11 @@ describe('ckan.modules.AutocompleteModule()', function () { if (jQuery.fn.select2) { this.select2 = sinon.stub(jQuery.fn, 'select2'); } else { - this.select2 = jQuery.fn.select2 = sinon.spy(); + this.select2 = jQuery.fn.select2 = sinon.stub().returns({ + data: sinon.stub().returns({ + on: sinon.stub() + }) + }); } this.el = document.createElement('input'); @@ -75,6 +79,8 @@ describe('ckan.modules.AutocompleteModule()', function () { createSearchChoice: this.module.formatTerm, // Not used by tags. initSelection: this.module.formatInitialValue }); + + it('should watch the keydown event on the select2 input'); }); }); @@ -214,7 +220,42 @@ describe('ckan.modules.AutocompleteModule()', function () { }); }); - describe('._onChange', function () { - it('should reassign value of the select2 element to ensure comma sepeated values are rendered correctly'); + describe('._onKeydown(event)', function () { + beforeEach(function () { + this.keyDownEvent = jQuery.Event("keydown", { which: 188 }); + this.fakeEvent = {}; + this.clock = sinon.useFakeTimers(); + this.jQuery = sinon.stub(jQuery.fn, 'init', jQuery.fn.init); + this.Event = sinon.stub(jQuery, 'Event').returns(this.fakeEvent); + this.trigger = sinon.stub(jQuery.fn, 'trigger'); + }); + + afterEach(function () { + this.clock.restore(); + this.jQuery.restore(); + this.Event.restore(); + this.trigger.restore(); + }); + + it('should trigger fake "return" keypress if a comma is pressed', function () { + this.module._onKeydown(this.keyDownEvent); + + this.clock.tick(100); + + assert.called(this.jQuery); + assert.called(this.Event); + assert.called(this.trigger); + assert.calledWith(this.trigger, this.fakeEvent); + }); + + it('should do nothing if another key is pressed', function () { + this.keyDownEvent.which = 200; + + this.module._onKeydown(this.keyDownEvent); + + this.clock.tick(100); + + assert.notCalled(this.Event); + }); }); }); From 2c1c4c7de28a90d5af6cac6836d908c9ff03f760 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 24 Jul 2012 12:37:21 +0100 Subject: [PATCH 074/103] [#2743] add dataset to group functionality --- ckan/controllers/package.py | 8 ++++++-- ckan/templates/group/read.html | 2 +- ckan/templates/package/snippets/package_basic_fields.html | 5 +++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 562fee6f7a0..a8f035f28f6 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -444,9 +444,8 @@ def new(self, data=None, errors=None, error_summary=None): return self._save_new(context) data = data or clean_dict(unflatten(tuplize_dict(parse_params( - request.params, ignore_keys=CACHE_PARAMETERS)))) + request.POST, ignore_keys=CACHE_PARAMETERS)))) c.resources_json = json.dumps(data.get('resources', [])) - # convert tags if not supplied in data if data and not data.get('tag_string'): data['tag_string'] = ', '.join( @@ -462,6 +461,11 @@ def new(self, data=None, errors=None, error_summary=None): elif data.get('state') == 'draft-complete': stage = ['active', 'complete', 'complete'] + # if we are creating from a group then this allows the group to be + # set automatically + data['group_id'] = request.params.get('group') or \ + request.params.get('groups__0__id') + vars = {'data': data, 'errors': errors, 'error_summary': error_summary, 'action': 'new', 'stage': stage} diff --git a/ckan/templates/group/read.html b/ckan/templates/group/read.html index 03d3d5fb92b..cd26d8b5bcb 100644 --- a/ckan/templates/group/read.html +++ b/ckan/templates/group/read.html @@ -9,7 +9,7 @@ {% block actions_content %} {% if h.check_access('group_update', {'id': c.group.id}) %} -
    • {% link_for _('Add Dataset'), controller='group', action='edit', id=c.group_dict.name, class_='btn', icon='plus' %}
      • +
    • {% link_for _('Add Dataset to Group'), controller='package', action='new', group=c.group_dict.id, class_='btn', icon='plus' %}
    • {% link_for _('Edit'), controller='group', action='edit', id=c.group_dict.name, class_='btn', icon='cog' %}
      • {% endif %} {#
    • {% link_for _('History'), controller='group', action='history', id=c.group_dict.name, class_='btn', icon='undo' %}
      • #} diff --git a/ckan/templates/package/snippets/package_basic_fields.html b/ckan/templates/package/snippets/package_basic_fields.html index e88b0d24363..994ab0149bc 100644 --- a/ckan/templates/package/snippets/package_basic_fields.html +++ b/ckan/templates/package/snippets/package_basic_fields.html @@ -32,3 +32,8 @@
      + +{# if we have a default group then this wants remembering #} +{% if data.group_id %} + +{% endif %} From dd4a78aaf28a90b121608f2fc723976a16e7b1f0 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 12:38:49 +0100 Subject: [PATCH 075/103] [#2695] Update the select2 styles --- ckan/public/base/css/main.css | 21 ++++++++++++++------- ckan/public/base/less/forms.less | 28 ++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 15 deletions(-) diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index dfa0c09c30b..5caf90d64de 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -5440,15 +5440,19 @@ textarea { background: url("../images/loading-spinner.gif") no-repeat center right; padding-right: 5px; } -.select2-container .select2-choice, -.select2-container-multi .select2-choices .select2-search-field input { - height: 29px; +.select2-container .select2-choice input, +.select2-container-multi .select2-choices .select2-search-field:first-child input { + font-size: 14px; +} +.select2-container .select2-choice input, +.select2-container-multi .select2-choices .select2-search-field:first-child input { + padding-left: 10px; } -.select2-container .select2-choice { - padding: 4px 8px 2px; +.select2-container { + margin-top: 5px; } -.select2-container .select2-choice div b { - background-position: 0 5px; +.select2-container-multi { + margin-top: 0; } .select2-container-multi .select2-choices .select2-search-choice { padding: 5px 8px 5px 22px; @@ -5491,6 +5495,9 @@ textarea { .select2-container-multi .select2-drop { margin-top: -2px; } +.control-full .select2-container { + width: 520px !important; +} .dataset-item { border-bottom: 1px dotted #ccc; padding-bottom: 20px; diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 5a793f5182d..368b1f86dc0 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -509,18 +509,24 @@ textarea { } // Select2 -.select2-container .select2-choice, -.select2-container-multi .select2-choices .select2-search-field input { - // Need to compensate for box-sizing: border-box - height: 29px; + +.select2-container .select2-choice input, +.select2-container-multi .select2-choices .select2-search-field:first-child input { + font-size: 14px; +} + +.select2-container .select2-choice input, +.select2-container-multi .select2-choices .select2-search-field:first-child input { + // Only apply if there are no other tags (ie. is the first child) + padding-left: 10px; } -.select2-container .select2-choice { - padding: 4px 8px 2px; +.select2-container { + margin-top: 5px; } -.select2-container .select2-choice div b { - background-position: 0 5px; +.select2-container-multi { + margin-top: 0; } .select2-container-multi .select2-choices .select2-search-choice { @@ -558,3 +564,9 @@ textarea { .select2-container-multi .select2-drop { margin-top: -2px; } + +.control-full .select2-container { + // Important is bad but select2 incorrectly calculates the width and + // sets it on the element. + width: 520px !important; +} From 0c723f7edfdaea50f9d3017c017d805e521cdccc Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 12:39:34 +0100 Subject: [PATCH 076/103] [#2695] Add select2 to all resource select boxes --- ckan/templates/package/snippets/package_basic_fields.html | 4 ++-- ckan/templates/package/snippets/package_metadata_fields.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/templates/package/snippets/package_basic_fields.html b/ckan/templates/package/snippets/package_basic_fields.html index 994ab0149bc..b54283d46ed 100644 --- a/ckan/templates/package/snippets/package_basic_fields.html +++ b/ckan/templates/package/snippets/package_basic_fields.html @@ -18,8 +18,8 @@ {% set error = errors.license_id %}
      - + {% for licence_desc, licence_id in licences|sort if licence_desc %} {% endfor %} diff --git a/ckan/templates/package/snippets/package_metadata_fields.html b/ckan/templates/package/snippets/package_metadata_fields.html index c7e2798f11c..af8b1fafbd4 100644 --- a/ckan/templates/package/snippets/package_metadata_fields.html +++ b/ckan/templates/package/snippets/package_metadata_fields.html @@ -6,7 +6,7 @@ {% set groups = h.dict_list_reduce(data.groups, 'id') %}
      - {% for group in groups_available %} From 233d4b76642b509c8d0962ae578517304eac6390 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 12:40:29 +0100 Subject: [PATCH 077/103] [#2695] Allow autocomplete module to be loaded on select boxes --- .../base/javascript/modules/autocomplete.js | 16 +++++++++------- .../base/test/spec/modules/autocomplete.spec.js | 1 - 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js index c85d9082f12..cbca28b237e 100644 --- a/ckan/public/base/javascript/modules/autocomplete.js +++ b/ckan/public/base/javascript/modules/autocomplete.js @@ -47,17 +47,19 @@ this.ckan.module('autocomplete', function (jQuery, _) { var settings = { formatResult: this.formatResult, formatNoMatches: this.formatNoMatches, - formatInputTooShort: this.formatInputTooShort, - createSearchChoice: this.formatTerm, // Not used by tags. - initSelection: this.formatInitialValue + formatInputTooShort: this.formatInputTooShort }; // Different keys are required depending on whether the select is // tags or generic completion. - if (this.options.tags) { - settings.tags = this._onQuery; - } else { - settings.query = this._onQuery; + if (!this.el.is('select')) { + if (this.options.tags) { + settings.tags = this._onQuery; + } else { + settings.query = this._onQuery; + settings.createSearchChoice = this.formatTerm; + } + settings.initSelection = this.formatInitialValue; } var select2 = this.el.select2(settings).data('select2'); diff --git a/ckan/public/base/test/spec/modules/autocomplete.spec.js b/ckan/public/base/test/spec/modules/autocomplete.spec.js index 96fec673b82..8e8c14a95c6 100644 --- a/ckan/public/base/test/spec/modules/autocomplete.spec.js +++ b/ckan/public/base/test/spec/modules/autocomplete.spec.js @@ -76,7 +76,6 @@ describe('ckan.modules.AutocompleteModule()', function () { formatResult: this.module.formatResult, formatNoMatches: this.module.formatNoMatches, formatInputTooShort: this.module.formatInputTooShort, - createSearchChoice: this.module.formatTerm, // Not used by tags. initSelection: this.module.formatInitialValue }); From bc930312a07c26b9a8b746234a5b93a27c597368 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 14:13:04 +0100 Subject: [PATCH 078/103] [#2737] Update placeholder text in resource format form --- ckan/templates/package/snippets/resource_form.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/snippets/resource_form.html b/ckan/templates/package/snippets/resource_form.html index 5ddbfbf4961..8471128542d 100644 --- a/ckan/templates/package/snippets/resource_form.html +++ b/ckan/templates/package/snippets/resource_form.html @@ -41,7 +41,7 @@ {{ form.markdown('description', id='field-description', label=_('Description'), placeholder=_('Some useful notes about the data'), value=data.description, error=errors.description) }} {% set format_attrs = {'data-module': 'autocomplete', 'data-module-source': '/api/2/util/resource/format_autocomplete?incomplete=?'} %} - {{ form.input('format', id='field-format', label=_('Format'), placeholder=_('eg. application/json'), value=data.format, error=errors.format, classes=['control-medium'], attrs=format_attrs) }} + {{ form.input('format', id='field-format', label=_('Format'), placeholder=_('eg. CSV, XML or JSON'), value=data.format, error=errors.format, classes=['control-medium'], attrs=format_attrs) }} {% endblock %} {% block metadata_fields %} From 205ef00e85b1a6b4efa884cf176fb32fc70ed0a2 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 14:18:50 +0100 Subject: [PATCH 079/103] [#2375] Fix tag field height --- ckan/public/base/css/main.css | 3 +++ ckan/public/base/less/forms.less | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 5caf90d64de..0305d192fb6 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -5444,6 +5444,9 @@ textarea { .select2-container-multi .select2-choices .select2-search-field:first-child input { font-size: 14px; } +.select2-container-multi .select2-choices .select2-search-field input { + height: 29px; +} .select2-container .select2-choice input, .select2-container-multi .select2-choices .select2-search-field:first-child input { padding-left: 10px; diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 368b1f86dc0..aa6dbe938b8 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -515,6 +515,10 @@ textarea { font-size: 14px; } +.select2-container-multi .select2-choices .select2-search-field input { + height: 29px; +} + .select2-container .select2-choice input, .select2-container-multi .select2-choices .select2-search-field:first-child input { // Only apply if there are no other tags (ie. is the first child) From 64935e8259e9b1524e0b119dc153c27f6be1cbb0 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 24 Jul 2012 14:37:33 +0100 Subject: [PATCH 080/103] [#2740] better breadcrumb on add resource to dataset --- ckan/templates/package/new_resource.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 94fb3acaff8..01c6b659605 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -23,3 +23,9 @@

      {{ _('What {% endblock %} + +{% block breadcrumb_content %} +
    • {% link_for _('Datasets'), controller='package', action='search', highlight_actions = 'new index' %}
      • +
    • {{ pkg_dict.name|truncate(35) }}
      • +
    • Add resource
      • +{% endblock %} From b661e751f3fbd999a869d8b4d1bc9ecc99d73531 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 14:45:43 +0100 Subject: [PATCH 081/103] [#2695] Improve responsiveness of tag form --- ckan/public/base/javascript/modules/autocomplete.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ckan/public/base/javascript/modules/autocomplete.js b/ckan/public/base/javascript/modules/autocomplete.js index cbca28b237e..b85672444f6 100644 --- a/ckan/public/base/javascript/modules/autocomplete.js +++ b/ckan/public/base/javascript/modules/autocomplete.js @@ -118,6 +118,8 @@ this.ckan.module('autocomplete', function (jQuery, _) { if (!this._debounced) { // Set a timer to prevent the search lookup occurring too often. this._debounced = setTimeout(function () { + var term = module._lastTerm; + delete module._debounced; // Cancel the previous request if it hasn't yet completed. @@ -125,8 +127,13 @@ this.ckan.module('autocomplete', function (jQuery, _) { module._last.abort(); } - module._last = module.getCompletions(module._lastTerm, fn); + module._last = module.getCompletions(term, function (terms) { + fn(module._lastResults = terms); + }); }, this.options.interval); + } else { + // Re-use the last set of terms. + fn(this._lastResults || {results: []}); } } else { fn({results: []}); From b3f0433b7e873379bbc9afae4067e4e81214c8d2 Mon Sep 17 00:00:00 2001 From: tobes Date: Tue, 24 Jul 2012 16:30:22 +0100 Subject: [PATCH 082/103] [#2375] set default license to not specified --- ckan/templates/package/snippets/package_basic_fields.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/snippets/package_basic_fields.html b/ckan/templates/package/snippets/package_basic_fields.html index b54283d46ed..83871be5f3d 100644 --- a/ckan/templates/package/snippets/package_basic_fields.html +++ b/ckan/templates/package/snippets/package_basic_fields.html @@ -20,7 +20,7 @@
      {% if error %}{{ error }}{% endif %} From dbc8766fd20ccc4ff627d6a11604694674e7eb77 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 24 Jul 2012 17:36:55 +0100 Subject: [PATCH 083/103] [#2693] Update requirements files in LICENSE.txt --- LICENSE.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 0e58e4e3401..4eb1394ef49 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -22,9 +22,9 @@ Note ==== CKAN is sometimes packaged directly with other software (listed in -requires/lucid_conflict.txt). In these cases, we are required to list the -licenses of the packaged softare too. They are all AGPL compatible and read as -follows in the next sections. +pip-requirements.txt, pip-requirements-test.txt and pip-requirements-docs.txt). +In these cases, we are required to list the licenses of the packaged softare +too. They are all AGPL compatible and read as follows in the next sections. WebHelpers ---------- From b31109b628a21072ec0c2dec2a5d5a1c2cc0ed9c Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Tue, 24 Jul 2012 18:11:30 +0100 Subject: [PATCH 084/103] [#2375] Add some front end documentation --- doc/coding-standards.rst | 47 ----- doc/frontend-development.rst | 391 +++++++++++++++++++++++++++++++++++ doc/install-from-source.rst | 36 +--- 3 files changed, 393 insertions(+), 81 deletions(-) create mode 100644 doc/frontend-development.rst diff --git a/doc/coding-standards.rst b/doc/coding-standards.rst index 32987bc6150..f9a88f9032e 100644 --- a/doc/coding-standards.rst +++ b/doc/coding-standards.rst @@ -833,53 +833,6 @@ comments that do not form part of the documentation. :: module.Broadcast = Broadcast; } -File Structure --------------- - -All public JavaScript files should be contained within a _javascript_ directory -within the _public_ directory and files should be structured accordingly. :: - - lib/ - main.js - utils.js - components/ - vendor/ - jquery.js - jquery.plugin.js - underscore.js - templates/ - test/ - index.html - spec/ - main-spec.js - utils-spec.js - vendor/ - mocha.js - mocha.css - chai.js - -All files and directories should be lowercase with hyphens used to separate words. - -lib - Should contain all application files. These can be structured appropriately. - It is recommended that *main.js* be used as the bootstrap filename that sets - up the page. - -vendor - Should contain all external dependencies. These should not contain - version numbers in the filename. This information should be available in - the header comment of the file. Library plugins should be prefixed with the - library name. eg the hover intent jQuery plugin would have the filename - *jquery.hover-intent.js*. - -templates - Should be stored in a seperate directory and have the .html - extension. -test - Contains the test runner *index.html*. *vendor* contains all test - dependencies and libraries. *spec* contains the actual test files. Each - test file should be the filename with *-spec* appended. - JSHint ------ diff --git a/doc/frontend-development.rst b/doc/frontend-development.rst new file mode 100644 index 00000000000..79a31076ae6 --- /dev/null +++ b/doc/frontend-development.rst @@ -0,0 +1,391 @@ +Install front end dependencies +============================== + +The front end stylesheets are written using +`LESS `_. This depends on +`node `_ being installed on the system. + +Instructions for installing node can be found on the `project +website `_. + +On ubuntu node can be installed via: + +:: + + $ apt-get install nodejs + +LESS can then be installed via the node package manager which is bundled +with node. ``cd`` into the ``pyenv/src/ckan`` and run: + +:: + + $ npm install less + +A watcher script can then be used to compile the CSS whenever the LESS +changes: + +:: + + $ ./bin/less + +This will generate a main.debug.css file. To build the production CSS to +commit into the repository run: + +:: + + $ ./bin/less --production + +This will generate the main.css file that should be committed for use in +packaged versions of CKAN. (This command will actually start a watch +script so you'll have to exit it with ``ctrl-c`` to actually commit the +file.) + +(If someone could move this into a paster script that would be great) + +File Structure +============== + +All front-end files to be served via a web server are located in the +``public`` directory (in the case of the new CKAN base theme it's +``public/base``). + +:: + + css/ + main.css + less/ + main.less + ckan.less + javascript/ + main.js + utils.js + components/ + vendor/ + jquery.js + jquery.plugin.js + underscore.js + bootstrap.css + test/ + index.html + spec/ + main.spec.js + utils.spec.js + vendor/ + mocha.js + mocha.css + chai.js + +All files and directories should be lowercase with hyphens used to +separate words. + +css + Should contain any site specific CSS files including compiled + production builds generated by LESS. +less + Should contain all the less files for the site. Additional vendor + styles should be added to the *vendor* directory and included in + main.less. +javascript + Should contain all website files. These can be structured + appropriately. It is recommended that *main.js* be used as the + bootstrap filename that sets up the page. +vendor + Should contain all external dependencies. These should not contain + version numbers in the filename. This information should be + available in the header comment of the file. Library plugins should + be prefixed with the library name. If a dependency has many files + (such as bootstrap) then the entire directory should be included as + distributed by the maintainer. +test + Contains the test runner *index.html*. *vendor* contains all test + dependencies and libraries. *spec* contains the actual test files. + Each test file should be the filename with *.spec* appended. + +Stylesheets +=========== + +Because all the stylesheets are using LESS we need to compile them +before beginning development. In production CKAN will look for the +``main.css`` file which is included in the repository. In development +CKAN looks for the file ``main.debug.css`` which you will need to +generate by running: + +:: + + $ ./bin/less + +This will watch for changes to all of the less files and automatically +rebuild the CSS for you. To quit the script press ``ctrl-c``. + +There are many LESS scripts which attempt to group the styles in useful +groups. + +The main two are: + +- main.less: This contains *all* the styles for the website including + dependancies and local styles. The only files that are excluded here + are those that are conditionally loaded such as IE only CSS and large + external apps (like recline) that only appear on a single page. +- ckan.less: This includes all the local ckan stylesheets. + +There is a very basic pattern primer available at: + +:: + + http://localhost:5000/test/primer/ + +This should contain common components and slowly be expanded to provide +documentation on the CKAN markup and styles. But for the moment it +serves as useful unit tests for the CSS. + +JavaScript +========== + +The core of the CKAN JavaScript is split up into three areas. + +- Core (such as i18n, pub/sub and API clients) +- Modules (small HTML components or widgets) +- jQuery Plugins (very small reusable components) + +Core +---- + +Everything in the CKAN application lives on the ``ckan`` namespace. +Currently there are four main components that make up the core. + +Modules +~~~~~~~ + +Modules are the core of the CKAN website, every component that is +interactive on the page should be a module. These are then initialized +by including a ``data-module`` attribute on an element on the page. + +The idea is to create small isolated components that can easily be +tested. They should ideally not use any global objects, all +functionality should be provided to them via a "sandbox" object. + +There is a global factory that can be used to create new modules. + +:: + + ckan.module('my-module', { + initialize: function () { + // Called when a module is created. + }, + teardown: function () { + // Called before a module is removed from the page. + } + }); + +jQuery and Localisation methods are available via +``this.sandbox.jQuery`` and ``this.sandbox.translate()`` respectively. +To save typing these two common objects we can take advantage of +JavaScript closures and use an alternative module syntax that accepts a +factory function. + +:: + + ckan.module('my-module', function (jQuery, translate) { + return { + initialize: function () { + // Called when a module is created. + // jQuery and translate are available here. + }, + teardown: function () { + // Called before a module is removed from the page. + } + } + }); + +Pub/Sub +~~~~~~~ + +There is a simple pub/sub module included under ``ckan.pubsub`` it's +methods are available to modules via +``this.sandbox.publish/subscribe/unsubscribe``. This can be used to +publish messages between modules. + +Client +~~~~~~ + +Ideally no module should use jQuery.ajax() to make XHR requests to the +CKAN API, all functionality should be provided via the client object. + +:: + + ckan.module('my-module', { + initialize: function () { + this.sandbox.client.getCompletions(this.options.completionsUrl); + } + }); + +i18n/Jed +~~~~~~~~ + +`Jed `_ is a Gettext implementation in +JavaScript. It is used throughout the application to create translatable +strings. An instance of Jed is available on the ``ckan.i18n`` object. + +Modules get access to the ``translate()`` function via both the initial +factory function and the ``this.sandbox.translate()`` object. + +:: + + ckan.module('my-module', function (jQuery, translate) { + return {/* my module */}; + }); + + ckan.module('my-module', { + initialize: function () { + this.sandbox.translate('my string'); + } + }); + +Modules +------- + +Life Cycle +~~~~~~~~~~ + +CKAN modules are intialised on document ready. The +``ckan.module.initialize()`` will look for all elements on the page with +a ``data-module`` attribute and attempt to create an instance. + +:: + + + +The module will be created with the element, an options object extracted +from ``data-module-*`` attributes and a new sandbox instance. + +Once created the modules ``initialize()`` method will be called allowing +the module to set themselves up. + +Modules should also provide a ``teardown()`` method this isn't used at +the moment except in the unit tests to restore state but may become +useful in the future. + +Talking to each other +~~~~~~~~~~~~~~~~~~~~~ + +Modules should use the publish/subscribe methods to talk to each other +and allow different areas of the UI to update where relevant. + +:: + + ckan.module('language-picker', { + initialize: function () { + var sandbox = this.sandbox; + this.el.on('change', function () { + sandbox.publish('change:lang', this.selected); + }); + } + }); + + ckan.module('language-notifier', { + initialize: function () { + this.sandbox.subscribe('change:lang', function (lang) { + alert('language is now ' + lang); + }); + } + }); + +Internationalisation +~~~~~~~~~~~~~~~~~~~~ + +All strings within modules should be internationalised. Strings can be +set in the ``options.i18n`` object and there is a ``.i18n()`` helper for +retrieving them. + +:: + + ckan.module('language-picker', function (jQuery, _) { + return { + options: { + i18n: { + hello: _('Hello') + } + }, + initialize: function () { + this.i18n('hello'); // "Hello" + } + } + }); + +String interpolation can be provided by passing extra arguments. + +:: + + ckan.module('language-picker', function (jQuery, _) { + return { + options: { + i18n: { + hello: _('Hello %s') + } + }, + initialize: function () { + var name = 'Dave'; + this.i18n('hello', name); // "Hello Dave" + } + } + }); + +Plural versions can also be provided using a function and the chained +Jed API. + +:: + + ckan.module('language-picker', function (jQuery, _) { + return { + options: { + i18n: { + apples: function (n) { + return _('I have %d apple').isPlural(n, 'I have %d apples'); + } + } + }, + initialize: function () { + var total = 1; + this.i18n('apples', total); // "I have 1 apple" + this.i18n('apples', 3); // "I have 3 apples" + } + } + }); + +jQuery Plug-ins +--------------- + +Any functionality that is not directly related to ckan should be +packaged up in a jQuery plug-in if possible. This keeps the modules +containing only ckan specific code and allows plug-ins to be reused on +other sites. + +Examples of these are ``jQuery.fn.slug()``, ``jQuery.fn.slugPreview()`` +and ``jQuery.proxyAll()``. + +Unit Tests +---------- + +There is currently a test suite available at: + +:: + + http://localhost:5000/base/test/index.html + +Every core component, module and plugin should have a set of unit tests. +Tests can be filtered using the ``grep={regexp}`` query string +parameter. + +:: + + http://localhost:5000/base/test/index.html?grep=^jQuery + +The libraries used for the tests are as follows. + +- `Mocha `_: A test runner using + a BDD style syntax. +- `Chai `_: An assertion library (we use the assert + style). +- `Sinon `_: A stubbing library, can stub objects, + timers and ajax requests. + diff --git a/doc/install-from-source.rst b/doc/install-from-source.rst index 8d014168e21..2d627f66e76 100644 --- a/doc/install-from-source.rst +++ b/doc/install-from-source.rst @@ -211,46 +211,14 @@ front page. the web browser on that same machine. For example run the textual web browser `w3m` in a separate ssh session to the one running `paster serve`. -10. Install front end dependencies -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The front end stylesheets are written using `LESS `_. This depends on `node `_ being -installed on the system. - -Instructions for installing node can be found on the `project website `_. - -On ubuntu node can be installed via:: - - $ apt-get install nodejs - -Less can then be installed via the node package manager which is bundled with -node. ``cd`` into the ``pyenv/src/ckan`` and run:: - - $ npm install less - -A watcher script can then be used to compile the CSS whenever the LESS -changes:: - - $ ./bin/less - -This will generate a main.debug.css file. To build the production CSS to commit -into the repository run:: - - $ ./bin/less --production - -This will generate the main.css file that should be committed for use in -packaged versions of CKAN. - -(If someone could move this into a paster script that would be great) - - -11. Run the CKAN Tests +10. Run the CKAN Tests ~~~~~~~~~~~~~~~~~~~~~~ Now that you've installed CKAN, you should run CKAN's tests to make sure that they all pass. See :doc:`test`. -12. You're done! +11. You're done! ~~~~~~~~~~~~~~~~ You can now proceed to :doc:`post-installation` which covers creating a CKAN From 40b6cca0a9dd7e6e55819d60c24532cedec3df85 Mon Sep 17 00:00:00 2001 From: Sean Hammond Date: Tue, 24 Jul 2012 17:31:17 +0100 Subject: [PATCH 085/103] Fix double counting of dataset and resource views Recent dataset views and total and recent resource views were being double counted by CKAN's internal page view tracking, fix so that each page view is counted only once (double-counting of total dataset page views was already fixed by a previous commit) --- ckan/lib/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py index 3aef7efe8e8..baaa4674ba9 100644 --- a/ckan/lib/cli.py +++ b/ckan/lib/cli.py @@ -977,13 +977,13 @@ def update_tracking(self, engine, summary_date): FROM tracking_summary t2 WHERE t1.url = t2.url AND t2.tracking_date <= t1.tracking_date - ) + t1.count + ) ,recent_views = ( SELECT sum(count) FROM tracking_summary t2 WHERE t1.url = t2.url AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14 - ) + t1.count + ) WHERE t1.running_total = 0 AND tracking_type = 'resource';''' engine.execute(sql) @@ -1000,7 +1000,7 @@ def update_tracking(self, engine, summary_date): FROM tracking_summary t2 WHERE t1.package_id = t2.package_id AND t2.tracking_date <= t1.tracking_date AND t2.tracking_date >= t1.tracking_date - 14 - ) + t1.count + ) WHERE t1.running_total = 0 AND tracking_type = 'page' AND t1.package_id IS NOT NULL AND t1.package_id != '~~not~found~~';''' From 37e02618b4761eb42ab95b815416802b0a796cce Mon Sep 17 00:00:00 2001 From: Ross Jones Date: Tue, 17 Jul 2012 12:11:22 +0100 Subject: [PATCH 086/103] Added support for specifying the sort direction Previously we only allowed the sort direction to be asc for groups names and desc for packages (package counts). Now users can specify a sort_direction to have a little more control. --- ckan/logic/action/get.py | 16 ++++---- ckan/tests/functional/test_group.py | 57 ++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a33e9c05830..e1e54b91639 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -287,6 +287,10 @@ def group_list(context, data_dict): raise logic.ParameterError('"order_by" value %r not implemented.' % order_by) all_fields = data_dict.get('all_fields',None) + # We want the default for packages to be highest first + default_order_dir = ("asc", "desc")[order_by == 'packages'] + order_direction = data_dict.get('order_direction', default_order_dir) + _check_access('group_list', context, data_dict) query = model.Session.query(model.Group).join(model.GroupRevision) @@ -295,16 +299,10 @@ def group_list(context, data_dict): if groups: query = query.filter(model.GroupRevision.name.in_(groups)) - if order_by == 'name': - sort_by, reverse = 'name', False - groups = query.all() - - if order_by == 'packages': - sort_by, reverse = 'packages', True - group_list = model_dictize.group_list_dictize(groups, context, - lambda x:x[sort_by], reverse) + lambda x:x[order_by], + order_direction == 'desc') if not all_fields: group_list = [group[ref_group_by] for group in group_list] @@ -1009,7 +1007,7 @@ def package_search(context, data_dict): returned datasets should begin. :type start: int :param qf: the dismax query fields to search within, including boosts. See - the `Solr Dismax Documentation + the `Solr Dismax Documentation `_ for further details. :type qf: string diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index ec12115d30f..4d997fc6221 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -7,7 +7,7 @@ from ckan import plugins import ckan.model as model from ckan.lib.create_test_data import CreateTestData -from ckan.logic import check_access, NotAuthorized +from ckan.logic import check_access, NotAuthorized, get_action from pylons import config @@ -99,6 +99,61 @@ def test_children(self): parent = model.Group.by_name("parent_group") assert_equal(len(parent.get_children_groups()), 0) + def test_sorting(self): + model.repo.rebuild_db() + + pkg1 = model.Package(name="pkg1") + pkg2 = model.Package(name="pkg2") + model.Session.add(pkg1) + model.Session.add(pkg2) + + CreateTestData.create_groups([{'name': "alpha", 'packages': []}, + {'name': "beta", + 'packages': ["pkg1", "pkg2"]}, + {'name': "delta", + 'packages': ["pkg1"]}, + {'name': "gamma", 'packages': []}], + admin_user_name='russianfan') + + context = {'model': model, 'session': model.Session, + 'user': 'russianfan', 'for_view': True, + 'with_private': False} + data_dict = {'all_fields': True} + results = get_action('group_list')(context, data_dict) + assert results[0]['name'] == u'alpha', results[0]['name'] + assert results[-1]['name'] == u'gamma', results[-1]['name'] + + # Test name reverse + data_dict = {'all_fields': True, 'order_by': 'name', 'order_direction': 'desc'} + results = get_action('group_list')(context, data_dict) + assert results[0]['name'] == u'gamma', results[0]['name'] + assert results[-1]['name'] == u'alpha', results[-1]['name'] + + # Test default (name) reverse + data_dict = {'all_fields': True, 'order_direction': 'desc'} + results = get_action('group_list')(context, data_dict) + assert results[0]['name'] == u'gamma', results[0]['name'] + assert results[-1]['name'] == u'alpha', results[-1]['name'] + + # Test packages reversed + data_dict = {'all_fields': True, 'order_by': 'packages', 'order_direction': 'desc'} + results = get_action('group_list')(context, data_dict) + assert results[0]['name'] == u'beta', results[0]['name'] + assert results[1]['name'] == u'delta', results[1]['name'] + + # Test packages forward + data_dict = {'all_fields': True, 'order_by': 'packages', 'order_direction': 'asc'} + results = get_action('group_list')(context, data_dict) + assert results[-2]['name'] == u'delta', results[-2]['name'] + assert results[-1]['name'] == u'beta', results[-1]['name'] + + # Default ordering for packages + data_dict = {'all_fields': True, 'order_by': 'packages'} + results = get_action('group_list')(context, data_dict) + assert results[0]['name'] == u'beta', results[0]['name'] + assert results[1]['name'] == u'delta', results[1]['name'] + + def test_mainmenu(self): # the home page does a package search so have to skip this test if # search is not supported From b4e5cecbc6b340d7615775cc350bafefa5e47c19 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 25 Jul 2012 09:52:46 +0100 Subject: [PATCH 087/103] change group_list sorting to use sort --- ckan/logic/action/get.py | 63 ++++++++++++++++++++++++----- ckan/tests/functional/test_group.py | 14 ++----- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index e1e54b91639..c7bf9f60a62 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -266,8 +266,12 @@ def group_list(context, data_dict): '''Return a list of the names of the site's groups. :param order_by: the field to sort the list by, must be ``'name'`` or - ``'packages'`` (optional, default: ``'name'``) + ``'packages'`` (optional, default: ``'name'``) Deprecated use sort. :type order_by: string + :param sort: sorting of the search results. Optional. Default: + "name asc" string of field name and sort-order. The allowed fields are + 'name' and 'packages' + :type sort: string :param groups: a list of names of the groups to return, if given only groups whose names are in this list will be returned (optional) :type groups: list of strings @@ -278,18 +282,30 @@ def group_list(context, data_dict): :rtype: list of strings ''' + model = context['model'] api = context.get('api_version') groups = data_dict.get('groups') ref_group_by = 'id' if api == 2 else 'name'; - order_by = data_dict.get('order_by', 'name') - if order_by not in set(('name', 'packages')): - raise logic.ParameterError('"order_by" value %r not implemented.' % order_by) - all_fields = data_dict.get('all_fields',None) - # We want the default for packages to be highest first - default_order_dir = ("asc", "desc")[order_by == 'packages'] - order_direction = data_dict.get('order_direction', default_order_dir) + sort = data_dict.get('sort', 'name') + # order_by deprecated in ckan 1.8 + # if it is supplied and sort isn't use order_by and raise a warning + order_by = data_dict.get('order_by') + if order_by: + log.warn('`order_by` deprecated please use `sort`') + if not data_dict.get('sort'): + sort = order_by + # if the sort is packages and no sort direction is supplied we want to do a + # reverse sort to maintain compatibility. + if sort.strip() == 'packages': + sort = 'packages desc' + + sort_info = _unpick_search(sort, + allowed_fields=['name', 'packages'], + total=1) + + all_fields = data_dict.get('all_fields', None) _check_access('group_list', context, data_dict) @@ -301,8 +317,8 @@ def group_list(context, data_dict): groups = query.all() group_list = model_dictize.group_list_dictize(groups, context, - lambda x:x[order_by], - order_direction == 'desc') + lambda x:x[sort_info[0][0]], + sort_info[0][1] == 'desc') if not all_fields: group_list = [group[ref_group_by] for group in group_list] @@ -2217,3 +2233,30 @@ def dashboard_activity_list_html(context, data_dict): ''' activity_stream = dashboard_activity_list(context, data_dict) return _activity_list_to_html(context, activity_stream) + + +def _unpick_search(sort, allowed_fields=None, total=None): + ''' This is a helper function that takes a sort string + eg 'name asc, last_modified desc' and returns a list of + split field order eg [('name', 'asc'), ('last_modified', 'desc')] + allowed_fields can limit which field names are ok. + total controls how many sorts can be specifed ''' + sorts = [] + split_sort = sort.split(',') + for part in split_sort: + split_part = part.strip().split() + field = split_part[0] + if len(split_part) > 1: + order = split_part[1].lower() + else: + order = 'asc' + if allowed_fields: + if field not in allowed_fields: + raise logic.ParameterError('Cannot sort by field `%s`' % field) + if order not in ['asc', 'desc']: + raise logic.ParameterError('Invalid sort direction `%s`' % order) + sorts.append((field, order)) + if total and len(sorts) > total: + raise logic.ParameterError( + 'Too many sort criteria provided only %s allowed' % total) + return sorts diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index 4d997fc6221..098ae250c00 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -124,31 +124,25 @@ def test_sorting(self): assert results[-1]['name'] == u'gamma', results[-1]['name'] # Test name reverse - data_dict = {'all_fields': True, 'order_by': 'name', 'order_direction': 'desc'} - results = get_action('group_list')(context, data_dict) - assert results[0]['name'] == u'gamma', results[0]['name'] - assert results[-1]['name'] == u'alpha', results[-1]['name'] - - # Test default (name) reverse - data_dict = {'all_fields': True, 'order_direction': 'desc'} + data_dict = {'all_fields': True, 'sort': 'name desc'} results = get_action('group_list')(context, data_dict) assert results[0]['name'] == u'gamma', results[0]['name'] assert results[-1]['name'] == u'alpha', results[-1]['name'] # Test packages reversed - data_dict = {'all_fields': True, 'order_by': 'packages', 'order_direction': 'desc'} + data_dict = {'all_fields': True, 'sort': 'packages desc'} results = get_action('group_list')(context, data_dict) assert results[0]['name'] == u'beta', results[0]['name'] assert results[1]['name'] == u'delta', results[1]['name'] # Test packages forward - data_dict = {'all_fields': True, 'order_by': 'packages', 'order_direction': 'asc'} + data_dict = {'all_fields': True, 'sort': 'packages asc'} results = get_action('group_list')(context, data_dict) assert results[-2]['name'] == u'delta', results[-2]['name'] assert results[-1]['name'] == u'beta', results[-1]['name'] # Default ordering for packages - data_dict = {'all_fields': True, 'order_by': 'packages'} + data_dict = {'all_fields': True, 'sort': 'packages'} results = get_action('group_list')(context, data_dict) assert results[0]['name'] == u'beta', results[0]['name'] assert results[1]['name'] == u'delta', results[1]['name'] From 9472a94327c20333303810fbac90d2f4ca97eba0 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 11:10:52 +0100 Subject: [PATCH 088/103] [#2744] Give the text in the select boxes breathing room --- ckan/public/base/css/main.css | 5 +++++ ckan/public/base/less/forms.less | 6 ++++++ 2 files changed, 11 insertions(+) diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 0305d192fb6..631a8db132b 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -5498,6 +5498,11 @@ textarea { .select2-container-multi .select2-drop { margin-top: -2px; } +.select2-container .select2-results li { + line-height: 18px; + padding-top: 4px; + padding-bottom: 4px; +} .control-full .select2-container { width: 520px !important; } diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index aa6dbe938b8..61ba7cc8e1f 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -569,6 +569,12 @@ textarea { margin-top: -2px; } +.select2-container .select2-results li { + line-height: 18px; + padding-top: 4px; + padding-bottom: 4px; +} + .control-full .select2-container { // Important is bad but select2 incorrectly calculates the width and // sets it on the element. From bc5577740cee2fa8ec6dcb053ba06765bbda3c54 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 11:19:23 +0100 Subject: [PATCH 089/103] [#2375] Update the language snippet to use select2 --- ckan/public/base/css/main.css | 13 +++++++------ ckan/public/base/less/footer.less | 14 ++++++++------ ckan/templates/snippets/language_selector.html | 6 +++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 631a8db132b..a23c4f63d74 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -6553,16 +6553,14 @@ li .icon-large:before { .site-footer:after { clear: both; } -.site-footer a { - color: #ffffff; -} +.site-footer, +.site-footer a, .site-footer label { - color: #ffffff; + color: #fff; + color: rgba(255, 255, 255, 0.8); } .footer-links, .attribution { - opacity: 0.8; - filter: alpha(opacity=80); float: left; margin-left: 20px; width: 460px; @@ -6587,6 +6585,9 @@ li .icon-large:before { margin-top: 2px; background: url("../images/ckan-logo-footer.png") no-repeat top left; } +.attribution .select2-drop { + color: #000; +} .profile-info { *zoom: 1; -webkit-border-radius: 3px 3px 0 0; diff --git a/ckan/public/base/less/footer.less b/ckan/public/base/less/footer.less index 24af65e26bc..21ed7160fd7 100644 --- a/ckan/public/base/less/footer.less +++ b/ckan/public/base/less/footer.less @@ -4,17 +4,15 @@ padding: 20px 0; } -.site-footer a { - color: @mastheadLinkColor; -} - +.site-footer, +.site-footer a, .site-footer label { - color: @mastheadLinkColor; + color: #fff; + color: rgba(255, 255, 255, 0.8); } .footer-links, .attribution { - .opacity(80); .makeColumn(6); } @@ -41,3 +39,7 @@ margin-top: 2px; background: url("@{imagePath}/ckan-logo-footer.png") no-repeat top left; } + +.attribution .select2-drop { + color: #000; +} diff --git a/ckan/templates/snippets/language_selector.html b/ckan/templates/snippets/language_selector.html index 0f75d3ffdf8..c6a6a5776be 100644 --- a/ckan/templates/snippets/language_selector.html +++ b/ckan/templates/snippets/language_selector.html @@ -2,10 +2,10 @@ {% set current_lang = request.environ.CKAN_LANG %}
      - {% for locale in h.get_available_locales() %} - {% endfor %} From 5c1a319edeec11590d2944438e1f728bc8d6baad Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 25 Jul 2012 12:12:39 +0100 Subject: [PATCH 090/103] [#2747] fix breadcrumb in new resource --- ckan/templates/package/new_resource.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 01c6b659605..2f809137276 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -26,6 +26,6 @@

      {{ _('What {% block breadcrumb_content %}
    • {% link_for _('Datasets'), controller='package', action='search', highlight_actions = 'new index' %}
      • -
    • {{ pkg_dict.name|truncate(35) }}
      • +
    • {% link_for h.dataset_display_name(pkg_dict)|truncate(30), controller='package', action='read', id=pkg_dict.name %}
    • Add resource
      • {% endblock %} From 4dcbb385eeae0797f631f470b960eec83641e795 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 25 Jul 2012 12:16:52 +0100 Subject: [PATCH 091/103] [#2375] broken title fix --- ckan/templates/package/new_resource.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 2f809137276..60de1fd0feb 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -1,6 +1,6 @@ {% extends "package/base_form_page.html" %} -{% block title %}{{ _('Add data to the dataset') }} - {{ super }}{% endblock %} +{% block title %}{{ _('Add data to the dataset') }} - {{ super() }}{% endblock %} {% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage %}{% endblock %} From 14b6c1f37cd43fc28dfaffea234e2a94a33482cc Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 12:59:00 +0100 Subject: [PATCH 092/103] [#2375] Add ability to autocomplete packages As a test this has been implemented on the edit group form. --- ckan/public/base/javascript/client.js | 41 +++++++++++++--- ckan/public/base/test/spec/client.spec.js | 49 ++++++++++++++++++- ckan/templates/group/snippets/group_form.html | 3 +- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/ckan/public/base/javascript/client.js b/ckan/public/base/javascript/client.js index e34e3ce4593..cf9df8becef 100644 --- a/ckan/public/base/javascript/client.js +++ b/ckan/public/base/javascript/client.js @@ -40,7 +40,9 @@ * the data into an array of strings. This also will remove duplicates * from the results (this is case insensitive). * - * data - The parsed JSON response from the server. + * data - The parsed JSON response from the server. + * options - An object of options for the method. + * objects: If true returns an object of results. * * Examples * @@ -50,7 +52,13 @@ * * Returns the parsed object. */ - parseCompletions: function (data) { + parseCompletions: function (data, options) { + if (typeof data === 'string') { + // Package completions are returned as a crazy string. So we handle + // them separately. + return this.parsePackageCompletions(data, options); + } + var map = {}; var raw = data.ResultSet && data.ResultSet.Result || {}; @@ -59,10 +67,11 @@ item = jQuery.trim(item); var lowercased = item.toLowerCase(); + var returnObject = options && options.objects === true; if (lowercased && !map[lowercased]) { map[lowercased] = 1; - return item; + return returnObject ? {id: item, text: item} : item; } return null; @@ -90,13 +99,29 @@ * Returns an object of item objects. */ parseCompletionsForPlugin: function (data) { - var items = this.parseCompletions(data); + return { + results: this.parseCompletions(data, {objects: true}) + }; + }, - items = jQuery.map(items, function (item) { - return {id: item, text: item}; + /* Parses the string returned by the package autocomplete endpoint which + * is a newline separated list of packages. Each package consists of + * a name and an id separated by a pipe (|) character. + * + * string - The string returned by the API. + * + * Returns an array of parsed packages. + */ + parsePackageCompletions: function (string, options) { + var packages = jQuery.trim(string).split('\n'); + var parsed = []; + + return jQuery.map(packages, function (pkg) { + var parts = pkg.split('|'); + var id = jQuery.trim(parts.pop() || ''); + var text = jQuery.trim(parts.join('|') || ''); + return options && options.objects === true ? {id: id, text: text} : id; }); - - return {results: items}; }, /* Requests config options for a file upload. diff --git a/ckan/public/base/test/spec/client.spec.js b/ckan/public/base/test/spec/client.spec.js index a2635b3a9da..25f76333bf9 100644 --- a/ckan/public/base/test/spec/client.spec.js +++ b/ckan/public/base/test/spec/client.spec.js @@ -64,7 +64,7 @@ describe('ckan.Client()', function () { }); - describe('.parseCompletions(data)', function () { + describe('.parseCompletions(data, options)', function () { it('should return a string of tags for a ResultSet collection', function () { var data = { ResultSet: { @@ -106,6 +106,34 @@ describe('ckan.Client()', function () { assert.deepEqual(target, ["Test"]); }); + + it('should return an array of objects if options.objects is true', function () { + var data = { + ResultSet: { + Result: [ + {"Format": "json"}, {"Format": "csv"}, {"Format": "text"} + ] + } + }; + + var target = this.client.parseCompletions(data, {objects: true}); + + assert.deepEqual(target, [ + {id: "json", text: "json"}, + {id: "csv", text: "csv"}, + {id: "text", text: "text"} + ]); + }); + + it('should call .parsePackageCompletions() id data is a string', function () { + var data = 'Name|id'; + var target = sinon.stub(this.client, 'parsePackageCompletions'); + + this.client.parseCompletions(data, {objects: true}); + + assert.called(target); + assert.calledWith(target, data); + }); }); describe('.parseCompletionsForPlugin(data)', function () { @@ -128,7 +156,26 @@ describe('ckan.Client()', function () { ] }); }); + }); + + describe('.parsePackageCompletions(string, options)', function () { + it('should parse the package completions string', function () { + var data = 'Package 1|package-1\nPackage 2|package-2\nPackage 3|package-3\n'; + var target = this.client.parsePackageCompletions(data); + + assert.deepEqual(target, ['package-1', 'package-2', 'package-3']); + }); + it('should return an object if options.object is true', function () { + var data = 'Package 1|package-1\nPackage 2|package-2\nPackage 3|package-3\n'; + var target = this.client.parsePackageCompletions(data, {objects: true}); + + assert.deepEqual(target, [ + {id: 'package-1', text: 'Package 1'}, + {id: 'package-2', text: 'Package 2'}, + {id: 'package-3', text: 'Package 3'} + ]); + }); }); describe('.getStorageAuth()', function () { diff --git a/ckan/templates/group/snippets/group_form.html b/ckan/templates/group/snippets/group_form.html index 460b313f6e1..4b4aed4926c 100644 --- a/ckan/templates/group/snippets/group_form.html +++ b/ckan/templates/group/snippets/group_form.html @@ -65,7 +65,8 @@ {% endif %} {% set dataset_name = 'packages__%s__name' % data.packages|length %} - {{ form.input(dataset_name, label=_('Add Dataset'), id="field-dataset", value=data[dataset_name]) }} + {% set dataset_attrs = {'data-module': 'autocomplete', 'data-module-source': '/dataset/autocomplete?q=?'} %} + {{ form.input(dataset_name, label=_('Add Dataset'), id="field-dataset", value=data[dataset_name], classes=['control-medium'], attrs=dataset_attrs) }} {% endblock %}
      From 1b41a9bc8ce7f0d77a5e2eb5ab2c44f44a61afc5 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 13:26:17 +0100 Subject: [PATCH 093/103] [#2720] Add padding around gravatars on user listing --- ckan/public/base/css/main.css | 3 ++- ckan/public/base/less/media.less | 3 ++- ckan/templates/user/list.html | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index a23c4f63d74..99bf50f47fb 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -4730,7 +4730,8 @@ ul.icons li .icon-large:before { line-height: 1.3em; margin: 5px 0; } -.media-image { +.media-image, +.media .gravatar { position: relative; float: left; margin-right: 10px; diff --git a/ckan/public/base/less/media.less b/ckan/public/base/less/media.less index 079aef6c1ef..bf4a6830555 100644 --- a/ckan/public/base/less/media.less +++ b/ckan/public/base/less/media.less @@ -20,7 +20,8 @@ margin: 5px 0; } -.media-image { +.media-image, +.media .gravatar { position: relative; float: left; margin-right: 10px; diff --git a/ckan/templates/user/list.html b/ckan/templates/user/list.html index 30afe0aa71d..913b8d33674 100644 --- a/ckan/templates/user/list.html +++ b/ckan/templates/user/list.html @@ -12,7 +12,7 @@

      {{ _('Users') }}

      {% for user in c.page.items %} - + {% endfor %}
      {{ h.linked_user(user['name'], maxlength=20) }}
      {{ h.linked_user(user['name'], maxlength=20) }}
      From 1a42b471d3daa23401898efa8c4f8f8f57d02d97 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 13:50:50 +0100 Subject: [PATCH 094/103] [#2375] Pull in config extras from ini file into base.html --- ckan/templates/base.html | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 58f20b37f6c..2d709111ab7 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -75,6 +75,11 @@ {% endblock %} + + {# defined in the config.ini under "ckan.template_head_end" #} + {% block head_extras %} + {{ config.get('ckan.template_head_end') | safe }} + {% endblock %} {# Allows custom attributes to be added to the tag #} @@ -107,5 +112,10 @@ #} {%- block scripts %} {% endblock -%} + + {# defined in the config.ini under "ckan.template_footer_end" #} + {% block body_extras %} + {{ config.get('ckan.template_footer_end') | safe }} + {% endblock %} From eca4c9e83f007ff290e3a1ff503e9b0e852faf2d Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 14:03:41 +0100 Subject: [PATCH 095/103] [#2375] Add some missing functionality from the config file * Favicon is now loaded from the config file * The "ckan.template_head_end" option is included * The "ckan.template_footer_end" option is included --- ckan/public/base/javascript/main.js | 10 +++++++--- ckan/templates/base.html | 6 ++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js index 9f65fb62b89..d65a845c749 100644 --- a/ckan/public/base/javascript/main.js +++ b/ckan/public/base/javascript/main.js @@ -8,12 +8,16 @@ this.ckan = this.ckan || {}; ckan.initialize = function () { var body = jQuery('body'); - var trailingSlash = /\/$/; var location = window.location; var root = location.protocol + location.host; - this.SITE_ROOT = (body.data('siteRoot') || root).replace(trailingSlash, ''); - this.LOCALE_ROOT = (body.data('localeRoot') || root).replace(trailingSlash, ''); + function getRootFromData(key) { + return (body.data(key) || root).replace(/\/$/, ''); + } + + this.SITE_ROOT = getRootFromData('siteRoot'); + this.LOCALE_ROOT = getRootFromData('localeRoot'); + this.API_ROOT = getRootFromData('apiRoot'); this.module.initialize(); }; diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 2d709111ab7..29329ce86db 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -56,7 +56,9 @@ such as rss feeds and favicons in the same way as the meta block. #} {% block links -%} - + {%- with favicon = config.get('ckan.favicon') or h.url_for_static('/base/images/ckan.ico') -%} + + {% endwith -%} {% endblock -%} {# @@ -83,7 +85,7 @@ {# Allows custom attributes to be added to the tag #} - + {# The page block allows you to add content to the page. Most of the time it is From 58394b686ae33272f08d172e2fd69e93415a4cbf Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 14:25:03 +0100 Subject: [PATCH 096/103] [#2375] Fix undefined config options in the base.html file --- ckan/templates/base.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 29329ce86db..4494092ef1a 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -80,12 +80,12 @@ {# defined in the config.ini under "ckan.template_head_end" #} {% block head_extras %} - {{ config.get('ckan.template_head_end') | safe }} + {{ config.get('ckan.template_head_end', '') | safe }} {% endblock %} {# Allows custom attributes to be added to the tag #} - + {# The page block allows you to add content to the page. Most of the time it is @@ -116,8 +116,8 @@ {% endblock -%} {# defined in the config.ini under "ckan.template_footer_end" #} - {% block body_extras %} - {{ config.get('ckan.template_footer_end') | safe }} - {% endblock %} + {% block body_extras -%} + {{ config.get('ckan.template_footer_end', '') | safe }} + {%- endblock %} From 3e120730d2949d7b8bab404dd7c9652214a92178 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 14:25:29 +0100 Subject: [PATCH 097/103] [#2375] Add support for alternative API endpoints to Client --- ckan/public/base/javascript/client.js | 29 +++++++++++++++++++---- ckan/public/base/javascript/main.js | 2 +- ckan/public/base/test/spec/client.spec.js | 29 ++++++++++++++++++++++- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/ckan/public/base/javascript/client.js b/ckan/public/base/javascript/client.js index cf9df8becef..12dbaef24ae 100644 --- a/ckan/public/base/javascript/client.js +++ b/ckan/public/base/javascript/client.js @@ -1,11 +1,30 @@ (function (ckan, jQuery) { - function Client() { + function Client(options) { + this.endpoint = options && options.endpoint || ''; jQuery.proxyAll(this, /parse/); } jQuery.extend(Client.prototype, { + /* Creates an API url from the path provided. If a fully qualified url + * is provided then this function just returns the input. + * + * path - A path to add the API domain to. + * + * Examples + * + * client.url('/datasets'); // http://api.example.com/datasets + * + * Returns an url string. + */ + url: function (path) { + if (!(/^https?:\/\//i).test(path)) { + path = this.endpoint + '/' + path.replace(/^\//, ''); + } + return path; + }, + /* Retrieves a list of auto-completions from one of the various endpoints * and normalises the results into an array of tags. * @@ -31,7 +50,7 @@ } var formatter = options && options.format || this.parseCompletions; - var request = jQuery.ajax({url: url}); + var request = jQuery.ajax({url: this.url(url)}); return request.pipe(formatter).promise(request).then(success, error); }, @@ -154,7 +173,7 @@ } return jQuery.ajax({ - url: '/api/storage/auth/form/' + key, + url: this.url('/api/storage/auth/form/' + key), success: success, error: error }); @@ -182,7 +201,7 @@ } return jQuery.ajax({ - url: '/api/storage/metadata/' + key, + url: this.url('/api/storage/metadata/' + key), success: success, error: error }); @@ -238,7 +257,7 @@ }); ckan.sandbox.setup(function (instance) { - instance.client = new Client(); + instance.client = new Client({endpoint: ckan.API_ROOT}); }); ckan.Client = Client; diff --git a/ckan/public/base/javascript/main.js b/ckan/public/base/javascript/main.js index d65a845c749..a9e0ce6af0d 100644 --- a/ckan/public/base/javascript/main.js +++ b/ckan/public/base/javascript/main.js @@ -9,7 +9,7 @@ this.ckan = this.ckan || {}; ckan.initialize = function () { var body = jQuery('body'); var location = window.location; - var root = location.protocol + location.host; + var root = location.protocol + '//' + location.host; function getRootFromData(key) { return (body.data(key) || root).replace(/\/$/, ''); diff --git a/ckan/public/base/test/spec/client.spec.js b/ckan/public/base/test/spec/client.spec.js index 25f76333bf9..503f8384e69 100644 --- a/ckan/public/base/test/spec/client.spec.js +++ b/ckan/public/base/test/spec/client.spec.js @@ -12,6 +12,33 @@ describe('ckan.Client()', function () { assert.instanceOf(target, Client); }); + it('should set the .endpoint property to options.endpoint', function () { + var client = new Client({endpoint: 'http://example.com'}); + assert.equal(client.endpoint, 'http://example.com'); + }); + + it('should default the endpoint to a blank string', function () { + assert.equal(this.client.endpoint, ''); + }); + + describe('.url(path)', function () { + beforeEach(function () { + this.client.endpoint = 'http://api.example.com'; + }); + + it('should return the path with the enpoint prepended', function () { + assert.equal(this.client.url('/api/endpoint'), 'http://api.example.com/api/endpoint'); + }); + + it('should normalise preceding slashes in the path', function () { + assert.equal(this.client.url('api/endpoint'), 'http://api.example.com/api/endpoint'); + }); + + it('should return the string if it already has a protocol', function () { + assert.equal(this.client.url('http://example.com/my/endpoint'), 'http://example.com/my/endpoint'); + }); + }); + describe('.getCompletions(url, options, success, error)', function () { beforeEach(function () { this.fakePiped = sinon.stub(jQuery.Deferred()); @@ -40,7 +67,7 @@ describe('ckan.Client()', function () { var target = this.client.getCompletions('url', success, error); assert.called(jQuery.ajax); - assert.calledWith(jQuery.ajax, {url: 'url'}); + assert.calledWith(jQuery.ajax, {url: '/url'}); assert.called(this.fakePiped.then); assert.calledWith(this.fakePiped.then, success, error); From 159815afddfecd46c20b69fb5bdff5675658430c Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 25 Jul 2012 14:12:12 +0100 Subject: [PATCH 098/103] [#2752] highlight groups main nav link on group index --- ckan/lib/helpers.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index a3c85e3d277..79c75e89ef4 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -328,12 +328,14 @@ def nav_named_link(*args, **kwargs): return _nav_named_link(*args, **kwargs) def _nav_named_link(text, name, **kwargs): + highlight_actions = kwargs.pop('highlight_actions','').split() + controller = kwargs.get('controller') return link_to( text, url_for(name, **kwargs), -# class_=('active' if -# c.action in highlight_actions -# else '') + class_=('active' if + c.action in highlight_actions and c.controller == controller + else '') ) def subnav_link(*args, **kwargs): @@ -414,7 +416,9 @@ def default_group_type(): 'search' : dict(controller='package', action='search', highlight_actions = 'index search'), - 'default_group': dict(name='%s_index' % default_group_type()), + 'default_group': dict(name='%s_index' % default_group_type(), + controller='group', + highlight_actions='index search'), 'about' : dict(controller='home', action='about'), 'login' : dict(controller='user', action='login'), 'register' : dict(controller='user', action='register'), From d41af4ce1bf77d2a9c23146b910a5810cf1c2048 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 15:22:22 +0100 Subject: [PATCH 099/103] [#1553] Do not show upload when not logged in Previously the upload form would be displayed when the user was creating a dataset and not logged in. This would cause the upload to fail. Instead we now remove the field. --- ckan/templates/package/new_resource.html | 8 +++++++- ckan/templates/package/resource_edit.html | 3 ++- ckan/templates/package/snippets/resource_form.html | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ckan/templates/package/new_resource.html b/ckan/templates/package/new_resource.html index 60de1fd0feb..3c61f19f40a 100644 --- a/ckan/templates/package/new_resource.html +++ b/ckan/templates/package/new_resource.html @@ -1,8 +1,14 @@ {% extends "package/base_form_page.html" %} +{% if c.userobj %} + {% set logged_in = true %} +{% else %} + {% set logged_in = false %} +{% endif %} + {% block title %}{{ _('Add data to the dataset') }} - {{ super() }}{% endblock %} -{% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage %}{% endblock %} +{% block form %}{% snippet 'package/snippets/resource_form.html', data=data, errors=errors, error_summary=error_summary, include_metadata=false, pkg_name=pkg_name, stage=stage, allow_upload=logged_in %}{% endblock %} {% block secondary_content %}
      diff --git a/ckan/templates/package/resource_edit.html b/ckan/templates/package/resource_edit.html index c2002667f60..9f48aa42bb3 100644 --- a/ckan/templates/package/resource_edit.html +++ b/ckan/templates/package/resource_edit.html @@ -11,4 +11,5 @@
    • {% link_for _('Edit Resource'), controller='package', action='edit_resource', id=pkg_dict.name, resource_id=res.id %}
      • {% endblock %} -{% block form %}{{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action) }}{% endblock %} +{# logged_in is defined in new_resource.html #} +{% block form %}{{ h.snippet('package/snippets/resource_edit_form.html', data=data, errors=errors, error_summary=error_summary, pkg_name=pkg_dict.name, form_action=c.form_action, allow_upload=logged_in) }}{% endblock %} diff --git a/ckan/templates/package/snippets/resource_form.html b/ckan/templates/package/snippets/resource_form.html index 8471128542d..67cf8ee23c6 100644 --- a/ckan/templates/package/snippets/resource_form.html +++ b/ckan/templates/package/snippets/resource_form.html @@ -18,7 +18,7 @@
      {% block basic_fields %} -
      +
      {# This block uses a slightly odd pattern. Unlike the rest of the radio buttons which are wrapped _inside_ the labels here we place the label From b55a3513cbcaca63a8580f4bd9cc08850416f041 Mon Sep 17 00:00:00 2001 From: Aron Carroll Date: Wed, 25 Jul 2012 15:27:10 +0100 Subject: [PATCH 100/103] [#2375] Tweak truncation in resource_read.html --- ckan/templates/package/resource_read.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index b55df88458d..5ff67581810 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -7,7 +7,7 @@ {% block breadcrumb_content %}
    • {% link_for _('Datasets'), controller='package', action='search', highlight_actions = 'new index' %}
      • -
    • {% link_for h.dataset_display_name(c.package)|truncate(20), controller='package', action='read', id=c.package.get('name') %}
      • +
    • {% link_for h.dataset_display_name(c.package)|truncate(15), controller='package', action='read', id=c.package.get('name') %}
    • {{ h.resource_display_name(res)|truncate(30) }}
      • {% endblock %} From 3ba5d03fadc741bf46edd7f1e14de14c36e00410 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 25 Jul 2012 15:09:50 +0100 Subject: [PATCH 101/103] [#2375] get link class rules to use common code --- ckan/lib/helpers.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 79c75e89ef4..65828c0e4d0 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -294,27 +294,34 @@ def _nav_link(text, controller, **kwargs): icon: name of ckan icon to use within the link condition: if False then no link is returned ''' + kwargs['controller'] = controller if kwargs.get('inner_span'): text = literal('') + text + literal('') - highlight_actions = kwargs.pop("highlight_actions", - kwargs["action"]).split() icon = kwargs.pop('icon', None) if icon: text = literal(' ' % icon) + text - active =(' active' if - c.controller == controller and c.action in highlight_actions - else '') - class_ = kwargs.pop('class_', '') + active + class_ = _link_class(kwargs) if kwargs.pop('condition', True): link = link_to( text, - url_for(controller=controller, **kwargs), + url_for(**kwargs), class_=class_ ) else: link = '' return link +def _link_class(kwargs): + ''' creates classes for the link_to calls ''' + highlight_actions = kwargs.pop('highlight_actions', + kwargs.get('action')).split() + if (c.controller == kwargs.get('controller') + and c.action in highlight_actions): + active = ' active' + else: + active = '' + return kwargs.pop('class_', '') + active + def nav_named_link(*args, **kwargs): # subnav_link() used to need c passing as the first arg # this is deprecated as pointless @@ -328,14 +335,11 @@ def nav_named_link(*args, **kwargs): return _nav_named_link(*args, **kwargs) def _nav_named_link(text, name, **kwargs): - highlight_actions = kwargs.pop('highlight_actions','').split() - controller = kwargs.get('controller') + class_ = _link_class(kwargs) return link_to( text, url_for(name, **kwargs), - class_=('active' if - c.action in highlight_actions and c.controller == controller - else '') + class_=class_ ) def subnav_link(*args, **kwargs): @@ -350,10 +354,12 @@ def subnav_link(*args, **kwargs): return _subnav_link(*args, **kwargs) def _subnav_link(text, action, **kwargs): + kwargs['action'] = action + class_ = _link_class(kwargs) return link_to( text, url_for(action=action, **kwargs), - class_=('active' if c.action == action else '') + class_=class_ ) def subnav_named_route(*args, **kwargs): @@ -370,10 +376,13 @@ def subnav_named_route(*args, **kwargs): def _subnav_named_route(text, routename, **kwargs): """ Generate a subnav element based on a named route """ + # FIXME this is the same as _nav_named_link + # they should be combined + class_ = _link_class(kwargs) return link_to( text, url_for(str(routename), **kwargs), - class_=('active' if c.action == kwargs['action'] else '') + class_=class_ ) def build_nav_main(*args): From e03c1e1feff44739b4e1cbd155b4277c56d94b54 Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 25 Jul 2012 15:15:50 +0100 Subject: [PATCH 102/103] [#2753] fix user breadcrumbs --- ckan/templates/user/list.html | 2 +- ckan/templates/user/read.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/user/list.html b/ckan/templates/user/list.html index 913b8d33674..b9c8dde360a 100644 --- a/ckan/templates/user/list.html +++ b/ckan/templates/user/list.html @@ -3,7 +3,7 @@ {% block title %}{{ _('All Users') }} - {{ super() }}{% endblock %} {% block breadcrumb_content %} -
    • {{ h.nav_link(_('active'), controller='user', action='index') }}
      • +
    • {{ h.nav_link(_('Users'), controller='user', action='index') }}
      • {% endblock %} {% block primary_content %} diff --git a/ckan/templates/user/read.html b/ckan/templates/user/read.html index 726685b7d04..01d7d61fbba 100644 --- a/ckan/templates/user/read.html +++ b/ckan/templates/user/read.html @@ -5,7 +5,7 @@ {% block title %}{{ user.display_name }} - {{ super() }}{% endblock %} {% block breadcrumb_content %} -
    • {% link_for _('User'), controller='user', action='index' %}
      • +
    • {% link_for _('Users'), controller='user', action='index' %}
      • {% with title = _('Your Profile') if c.is_myself else user.display_name %}
    • {% link_for title|truncate(35), controller='user', action='read', id=user.name %}
      • {% endwith %} From 19de45dc50f923e3fd0e2a9177086d0a8a58692c Mon Sep 17 00:00:00 2001 From: tobes Date: Wed, 25 Jul 2012 15:30:02 +0100 Subject: [PATCH 103/103] [#2754] add plus icons on dataset read --- ckan/templates/package/read.html | 2 +- ckan/templates/snippets/related.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/templates/package/read.html b/ckan/templates/package/read.html index 645bca4eabb..da89ef5ee49 100644 --- a/ckan/templates/package/read.html +++ b/ckan/templates/package/read.html @@ -18,7 +18,7 @@
    • {% link_for _('Edit'), controller='package', action='edit', id=pkg.name, class_='btn', icon='wrench' %}
      • {% endif %} {% if h.check_access('package_update', {'id':pkg.id }) %} -
    • {% link_for _('Add resource'), controller='package', action='new_resource', id=pkg.name, class_='btn' %}
      • +
    • {% link_for _('Add resource'), controller='package', action='new_resource', id=pkg.name, class_='btn', icon='plus' %}
      • {% endif %} {% endblock %} diff --git a/ckan/templates/snippets/related.html b/ckan/templates/snippets/related.html index 858e3be1db4..8ea5f39de20 100644 --- a/ckan/templates/snippets/related.html +++ b/ckan/templates/snippets/related.html @@ -14,7 +14,7 @@

      {{ item.title }}

      {% else %}

      No apps, ideas, news stories or images have been related to this dataset yet, why not add one now?

      -

      Add Item

      +

      {% link_for _('Add Item'), controller='related', action='new', id=pkg_name, icon='plus', class_='btn' %}

      {% endif %}