From 2b75455ab392ff4f6dee649f52e3af51db81f3f1 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 24 Jan 2017 18:11:08 -0500 Subject: [PATCH 01/25] [#3414] package_search: return field info --- ckanext/datastore/db.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 140b09556fe..51acecb1612 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -226,6 +226,24 @@ def _guess_type(field): return 'text' +def _get_field_info(connection, resource_id): + u'''return a dictionary mapping column names to their info data, + when present''' + qtext = sqlalchemy.text(u''' + select pa.attname as name, pd.description as info + from pg_class pc, pg_attribute pa, pg_description pd + where pa.attrelid = pc.oid and pd.objoid = pc.oid + and pd.objsubid = pa.attnum and pc.relname = :res_id + and pa.attnum > 0 + ''') + try: + return dict( + (n, json.loads(v)) for (n, v) in + connection.execute(qtext, res_id=resource_id).fetchall()) + except ValueError: # don't die on non-json comments + return {} + + def _get_fields(context, data_dict): fields = [] all_fields = context['connection'].execute( @@ -993,7 +1011,8 @@ def search_data(context, data_dict): results = _execute_single_statement(context, sql_string, where_values) _insert_links(data_dict, limit, offset) - return format_results(context, results, data_dict) + return format_results(context, results, data_dict, _get_field_info( + context['connection'], data_dict['resource_id'])) def _execute_single_statement(context, sql_string, where_values): @@ -1007,13 +1026,16 @@ def _execute_single_statement(context, sql_string, where_values): return results -def format_results(context, results, data_dict): +def format_results(context, results, data_dict, field_info=None): result_fields = [] for field in results.cursor.description: - result_fields.append({ + f = { 'id': field[0].decode('utf-8'), 'type': _get_type(context, field[1]) - }) + } + if field_info and f['id'] in field_info: + f['info'] = field_info[f['id']] + result_fields.append(f) if len(result_fields) and result_fields[-1]['id'] == '_full_count': result_fields.pop() # remove _full_count From 685842ea036fa8b4510d304406db6f964ebf2d81 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Tue, 24 Jan 2017 23:08:58 -0500 Subject: [PATCH 02/25] [#3414] datastore_create: set/update field info --- ckanext/datastore/db.py | 50 ++++++++++++++++++++++++------- ckanext/datastore/logic/schema.py | 3 +- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/ckanext/datastore/db.py b/ckanext/datastore/db.py index 51acecb1612..68df4b5136c 100644 --- a/ckanext/datastore/db.py +++ b/ckanext/datastore/db.py @@ -357,15 +357,26 @@ def create_table(context, data_dict): }) fields = datastore_fields + supplied_fields + extra_fields - sql_fields = u", ".join([u'"{0}" {1}'.format( - f['id'], f['type']) for f in fields]) + sql_fields = u", ".join([u'{0} {1}'.format( + datastore_helpers.identifier(f['id']), f['type']) for f in fields]) - sql_string = u'CREATE TABLE "{0}" ({1});'.format( - data_dict['resource_id'], + sql_string = u'CREATE TABLE {0} ({1});'.format( + datastore_helpers.identifier(data_dict['resource_id']), sql_fields ) - context['connection'].execute(sql_string.replace('%', '%%')) + info_sql = [] + for f in supplied_fields: + info = f.get(u'info') + if isinstance(info, dict): + info_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format( + datastore_helpers.identifier(data_dict['resource_id']), + datastore_helpers.identifier(f['id']), + datastore_helpers.literal_string( + json.dumps(info, ensure_ascii=False)))) + + context['connection'].execute( + (sql_string + u';'.join(info_sql)).replace(u'%', u'%%')) def _get_aliases(context, data_dict): @@ -625,12 +636,29 @@ def alter_table(context, data_dict): 'type': _guess_type(records[0][field_id]) }) - for field in new_fields: - sql = 'ALTER TABLE "{0}" ADD "{1}" {2}'.format( - data_dict['resource_id'], - field['id'], - field['type']) - context['connection'].execute(sql.replace('%', '%%')) + alter_sql = [] + for f in new_fields: + alter_sql.append(u'ALTER TABLE {0} ADD {1} {2};'.format( + datastore_helpers.identifier(data_dict['resource_id']), + datastore_helpers.identifier(f['id']), + f['type'])) + + for f in supplied_fields: + if u'info' in f: + info = f.get(u'info') + if isinstance(info, dict): + info_sql = datastore_helpers.literal_string( + json.dumps(info, ensure_ascii=False)) + else: + info_sql = 'NULL' + alter_sql.append(u'COMMENT ON COLUMN {0}.{1} is {2}'.format( + datastore_helpers.identifier(data_dict['resource_id']), + datastore_helpers.identifier(f['id']), + info_sql)) + + if alter_sql: + context['connection'].execute( + u';'.join(alter_sql).replace(u'%', u'%%')) def insert_data(context, data_dict): diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index e445a8deb6e..2d7d7f69ae4 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -101,7 +101,8 @@ def datastore_create_schema(): 'aliases': [ignore_missing, list_of_strings_or_string], 'fields': { 'id': [not_empty, unicode], - 'type': [ignore_missing] + 'type': [ignore_missing], + 'info': [ignore_missing], }, 'primary_key': [ignore_missing, list_of_strings_or_string], 'indexes': [ignore_missing, list_of_strings_or_string], From 10b60d555df30a4be8240450d031fbc6079084ca Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 27 Jan 2017 13:12:56 -0500 Subject: [PATCH 03/25] [#3414] remove if datapusher in g.plugins from core templates --- ckan/templates/package/resource_edit_base.html | 4 +--- ckanext/datapusher/plugin.py | 4 ++++ ckanext/datapusher/templates/package/resource_edit_base.html | 5 +++++ 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 ckanext/datapusher/templates/package/resource_edit_base.html diff --git a/ckan/templates/package/resource_edit_base.html b/ckan/templates/package/resource_edit_base.html index ce991239664..70313c94d7a 100644 --- a/ckan/templates/package/resource_edit_base.html +++ b/ckan/templates/package/resource_edit_base.html @@ -22,9 +22,7 @@ {% block content_primary_nav %} {{ h.build_nav_icon('resource_edit', _('Edit resource'), id=pkg.name, resource_id=res.id) }} - {% if 'datapusher' in g.plugins %} - {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} - {% endif %} + {% block inner_primary_nav %}{% endblock %} {{ h.build_nav_icon('views', _('Views'), id=pkg.name, resource_id=res.id) }} {% endblock %} diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index a7db0f66aa7..553609f1f3f 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -72,6 +72,7 @@ def resource_data(self, id, resource_id): class DatapusherPlugin(p.SingletonPlugin): + p.implements(p.IConfigurer, inherit=True) p.implements(p.IConfigurable, inherit=True) p.implements(p.IActions) p.implements(p.IAuthFunctions) @@ -83,6 +84,9 @@ class DatapusherPlugin(p.SingletonPlugin): legacy_mode = False resource_show_action = None + def update_config(self, config): + p.toolkit.add_template_directory(config, 'templates') + def configure(self, config): self.config = config diff --git a/ckanext/datapusher/templates/package/resource_edit_base.html b/ckanext/datapusher/templates/package/resource_edit_base.html new file mode 100644 index 00000000000..7c45a5bab08 --- /dev/null +++ b/ckanext/datapusher/templates/package/resource_edit_base.html @@ -0,0 +1,5 @@ +{% ckan_extends %} + +{% block inner_primary_nav %} + {{ h.build_nav_icon('resource_data', _('DataPusher'), id=pkg.name, resource_id=res.id) }} +{% endblock %} From f6d0669b3cd19e3d090c255a71a22ee7874d27d1 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 27 Jan 2017 13:19:26 -0500 Subject: [PATCH 04/25] [#3414] resource_data is a datapusher template --- {ckan => ckanext/datapusher}/templates/package/resource_data.html | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {ckan => ckanext/datapusher}/templates/package/resource_data.html (100%) diff --git a/ckan/templates/package/resource_data.html b/ckanext/datapusher/templates/package/resource_data.html similarity index 100% rename from ckan/templates/package/resource_data.html rename to ckanext/datapusher/templates/package/resource_data.html From b352b74a1902c16c93755a1765ca765294f012ff Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 29 Jan 2017 18:17:31 -0500 Subject: [PATCH 05/25] [#3414] resource_data is a datapusher template --- ckanext/datapusher/plugin.py | 2 +- .../templates/{package => datapusher}/resource_data.html | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ckanext/datapusher/templates/{package => datapusher}/resource_data.html (100%) diff --git a/ckanext/datapusher/plugin.py b/ckanext/datapusher/plugin.py index 553609f1f3f..762bb7317ea 100644 --- a/ckanext/datapusher/plugin.py +++ b/ckanext/datapusher/plugin.py @@ -67,7 +67,7 @@ def resource_data(self, id, resource_id): except logic.NotAuthorized: base.abort(403, _('Not authorized to see this page')) - return base.render('package/resource_data.html', + return base.render('datapusher/resource_data.html', extra_vars={'status': datapusher_status}) diff --git a/ckanext/datapusher/templates/package/resource_data.html b/ckanext/datapusher/templates/datapusher/resource_data.html similarity index 100% rename from ckanext/datapusher/templates/package/resource_data.html rename to ckanext/datapusher/templates/datapusher/resource_data.html From c728ea428883390451be346715f611015cda1c7e Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 29 Jan 2017 19:52:37 -0500 Subject: [PATCH 06/25] [#3414] display Data Dictionary form --- .../templates/package/resource_edit_base.html | 1 + ckanext/datastore/controller.py | 43 +++++++++++++++++++ ckanext/datastore/plugin.py | 11 +++-- .../templates/datastore/dictionary.html | 27 ++++++++++++ .../templates/package/resource_edit_base.html | 6 +++ 5 files changed, 85 insertions(+), 3 deletions(-) create mode 100644 ckanext/datastore/templates/datastore/dictionary.html create mode 100644 ckanext/datastore/templates/package/resource_edit_base.html diff --git a/ckanext/datapusher/templates/package/resource_edit_base.html b/ckanext/datapusher/templates/package/resource_edit_base.html index 7c45a5bab08..9ae3dda1fdb 100644 --- a/ckanext/datapusher/templates/package/resource_edit_base.html +++ b/ckanext/datapusher/templates/package/resource_edit_base.html @@ -1,5 +1,6 @@ {% ckan_extends %} {% block inner_primary_nav %} + {{ super() }} {{ h.build_nav_icon('resource_data', _('DataPusher'), id=pkg.name, resource_id=res.id) }} {% endblock %} diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index a2bd391a6ea..7b9627c9919 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -1,12 +1,14 @@ # encoding: utf-8 import StringIO +import md5 import pylons from ckan.plugins.toolkit import ( Invalid, ObjectNotFound, + NotAuthorized, get_action, get_validator, _, @@ -14,6 +16,8 @@ response, BaseController, abort, + render, + c, ) from ckanext.datastore.writer import ( csv_writer, @@ -84,3 +88,42 @@ def result_page(offset, limit): limit -= PAGINATE_BY result = result_page(offset, limit) + + def dictionary(self, id, resource_id): + u'''data dictionary view: show/edit field labels and descriptions''' + if request.method == 'POST': + return + + try: + # resource_edit_base template uses these + c.pkg_dict = get_action('package_show')( + None, {'id': id}) + c.resource = get_action('resource_show')( + None, {'id': resource_id}) + rec = get_action('datastore_search')(None, { + 'resource_id': resource_id, + 'limit': 0}) + except (ObjectNotFound, NotAuthorized): + abort(404, _('Resource not found')) + + fields = [f for f in rec['fields'] if not f['id'].startswith('_')] + + return render('datastore/dictionary.html', + extra_vars={'fields': fields}) + + +def short_hash(field): + u''' + return a short hash (20 hex digits) of a json-compatible object. + + We're using this to identify fields modified by the user when they + submit a form, and to detect if the source data changed before the + user submitted their changes. There's no need for a strong + cryptographic hash because this hash is only for preventing + accidental conflicts not enforcing access controls. + ''' + return md5.md5(json.dumps( + field, + sort_keys=True, + ensure_ascii=False, + separators=(u',', u':').encode('utf-8'))).hexdigest()[:20] diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 073ffcee214..067d268a9e4 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -246,9 +246,14 @@ def get_auth_functions(self): 'datastore_change_permissions': auth.datastore_change_permissions} def before_map(self, m): - m.connect('/datastore/dump/{resource_id}', - controller='ckanext.datastore.controller:DatastoreController', - action='dump') + m.connect( + '/datastore/dump/{resource_id}', + controller='ckanext.datastore.controller:DatastoreController', + action='dump') + m.connect( + 'resource_dictionary', '/dataset/{id}/dictionary/{resource_id}', + controller='ckanext.datastore.controller:DatastoreController', + action='dictionary', ckan_icon='book') return m def before_show(self, resource_dict): diff --git a/ckanext/datastore/templates/datastore/dictionary.html b/ckanext/datastore/templates/datastore/dictionary.html new file mode 100644 index 00000000000..ec43183be56 --- /dev/null +++ b/ckanext/datastore/templates/datastore/dictionary.html @@ -0,0 +1,27 @@ +{% extends "package/resource_edit_base.html" %} + +{% import 'macros/form.html' as form %} + +{% block subtitle %}{{ h.dataset_display_name(pkg) }} - {{ h.resource_display_name(res) }}{% endblock %} + +{% block primary_content_inner %} + + {% set action = h.url_for( + controller='ckanext.datastore.plugin:DatastoreController', + action='dictionary', id=pkg.name, resource_id=res.id) %} + +
+ {% for f in fields %} +

{{ _( "Field {num}.").format(num=loop.index) }} {{ f.id }} ({{ f.type }})

+ {{ form.input('f' ~ loop.index ~ 'label', + label=_('Label'), id='field-f' ~ loop.index ~ 'label', + value=f.get('info', {}).get('label', ''), classes=['control-full']) }} + {{ form.markdown('f' ~ loop.index ~ 'notes', + label=_('Description'), id='field-d' ~ loop.index ~ 'notes', + value=f.get('info', {}).get('notes', '')) }} + {% endfor %} + +
+{% endblock %} diff --git a/ckanext/datastore/templates/package/resource_edit_base.html b/ckanext/datastore/templates/package/resource_edit_base.html new file mode 100644 index 00000000000..1baa8e0b134 --- /dev/null +++ b/ckanext/datastore/templates/package/resource_edit_base.html @@ -0,0 +1,6 @@ +{% ckan_extends %} + +{% block inner_primary_nav %} + {{ super() }} + {{ h.build_nav_icon('resource_dictionary', _('Data Dictionary'), id=pkg.name, resource_id=res.id) }} +{% endblock %} From 209204257f4b3c30e388ee632b528c9525cf755e Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 29 Jan 2017 20:20:13 -0500 Subject: [PATCH 07/25] [#3414] working Data Dictionary form --- ckanext/datastore/controller.py | 26 ++++++++++++++++--- .../templates/datastore/dictionary.html | 22 +++++++++------- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 7b9627c9919..2267921e3e6 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -18,6 +18,7 @@ abort, render, c, + h, ) from ckanext.datastore.writer import ( csv_writer, @@ -91,8 +92,6 @@ def result_page(offset, limit): def dictionary(self, id, resource_id): u'''data dictionary view: show/edit field labels and descriptions''' - if request.method == 'POST': - return try: # resource_edit_base template uses these @@ -105,10 +104,29 @@ def dictionary(self, id, resource_id): 'limit': 0}) except (ObjectNotFound, NotAuthorized): abort(404, _('Resource not found')) - + fields = [f for f in rec['fields'] if not f['id'].startswith('_')] - return render('datastore/dictionary.html', + if request.method == 'POST': + get_action('datastore_create')(None, { + 'resource_id': resource_id, + 'force': True, + 'fields': [{ + 'id': f['id'], + 'type': f['type'], + 'info': { + 'label': request.POST.get('f{0}label'.format(i)), + 'notes': request.POST.get('f{0}notes'.format(i)), + }} for i, f in enumerate(fields, 1)]}) + + h.redirect_to( + controller='ckanext.datastore.controller:DatastoreController', + action='dictionary', + id=id, + resource_id=resource_id) + + return render( + 'datastore/dictionary.html', extra_vars={'fields': fields}) diff --git a/ckanext/datastore/templates/datastore/dictionary.html b/ckanext/datastore/templates/datastore/dictionary.html index ec43183be56..567904275d3 100644 --- a/ckanext/datastore/templates/datastore/dictionary.html +++ b/ckanext/datastore/templates/datastore/dictionary.html @@ -7,19 +7,21 @@ {% block primary_content_inner %} {% set action = h.url_for( - controller='ckanext.datastore.plugin:DatastoreController', + controller='ckanext.datastore.controller:DatastoreController', action='dictionary', id=pkg.name, resource_id=res.id) %}
- {% for f in fields %} -

{{ _( "Field {num}.").format(num=loop.index) }} {{ f.id }} ({{ f.type }})

- {{ form.input('f' ~ loop.index ~ 'label', - label=_('Label'), id='field-f' ~ loop.index ~ 'label', - value=f.get('info', {}).get('label', ''), classes=['control-full']) }} - {{ form.markdown('f' ~ loop.index ~ 'notes', - label=_('Description'), id='field-d' ~ loop.index ~ 'notes', - value=f.get('info', {}).get('notes', '')) }} - {% endfor %} + {% block dictionary_form %} + {% for f in fields %} +

{{ _( "Field {num}.").format(num=loop.index) }} {{ f.id }} ({{ f.type }})

+ {{ form.input('f' ~ loop.index ~ 'label', + label=_('Label'), id='field-f' ~ loop.index ~ 'label', + value=f.get('info', {}).get('label', ''), classes=['control-full']) }} + {{ form.markdown('f' ~ loop.index ~ 'notes', + label=_('Description'), id='field-d' ~ loop.index ~ 'notes', + value=f.get('info', {}).get('notes', '')) }} + {% endfor %} + {% endblock %} From 28234dc3293a9df91dbe9f45b97166ae2fe412ba Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Thu, 2 Feb 2017 12:45:22 -0500 Subject: [PATCH 08/25] allow package_search to return id/name only instead of full dataset dictionary --- ckan/logic/action/get.py | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 04fa8dfeb4f..32ac4cc968b 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1833,6 +1833,8 @@ def package_search(context, data_dict): The parameter that controls which fields are returned in the solr query cannot be changed. CKAN always returns the matched datasets as dictionary objects. + if fl='id' or 'name', only this field in the dataset dictionary will + be returned. ''' # sometimes context['schema'] is None schema = (context.get('schema') or @@ -1875,8 +1877,10 @@ def package_search(context, data_dict): else: data_source = 'validated_data_dict' data_dict.pop('use_default_schema', None) + # return a list of package ids - data_dict['fl'] = 'id {0}'.format(data_source) + if data_dict.get('fl') not in ('id', 'name'): + data_dict['fl'] = 'id {0}'.format(data_source) # Remove before these hit solr FIXME: whitelist instead include_private = asbool(data_dict.pop('include_private', False)) @@ -1903,21 +1907,25 @@ def package_search(context, data_dict): # Add them back so extensions can use them on after_search data_dict['extras'] = extras - for package in query.results: - # get the package object - package_dict = package.get(data_source) - ## use data in search index if there - if package_dict: - # the package_dict still needs translating when being viewed - package_dict = json.loads(package_dict) - if context.get('for_view'): - for item in plugins.PluginImplementations( - plugins.IPackageController): - package_dict = item.before_view(package_dict) - results.append(package_dict) - else: - log.error('No package_dict is coming from solr for package ' - 'id %s', package['id']) + if data_dict.get('fl') in ['id', 'name']: + for package in query.results: + results.append( {data_dict.get('fl'):package} ) + else: + for package in query.results: + # get the package object + package_dict = package.get(data_source) + ## use data in search index if there + if package_dict: + # the package_dict still needs translating when being viewed + package_dict = json.loads(package_dict) + if context.get('for_view'): + for item in plugins.PluginImplementations( + plugins.IPackageController): + package_dict = item.before_view(package_dict) + results.append(package_dict) + else: + log.error('No package_dict is coming from solr for package ' + 'id %s', package['id']) count = query.count facets = query.facets From 8541f225c752ab33de9bc9e0806c1e33c4f360cd Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Fri, 3 Feb 2017 18:20:05 -0500 Subject: [PATCH 09/25] [#3414] Data Dictionary display on resource page --- ckanext/datastore/helpers.py | 15 ++++++++ ckanext/datastore/plugin.py | 5 +++ .../templates/package/resource_edit_base.html | 4 ++- .../templates/package/resource_read.html | 35 +++++++++++++++++-- 4 files changed, 56 insertions(+), 3 deletions(-) diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py index 02a6f5aac84..406e1bcc5fc 100644 --- a/ckanext/datastore/helpers.py +++ b/ckanext/datastore/helpers.py @@ -6,6 +6,8 @@ import paste.deploy.converters as converters import sqlparse +from ckan.plugins.toolkit import get_action, ObjectNotFound, NotAuthorized + log = logging.getLogger(__name__) @@ -107,3 +109,16 @@ def identifier(s): Return s as a quoted postgres identifier """ return u'"' + s.replace(u'"', u'""').replace(u'\0', '') + u'"' + + +def datastore_dictionary(resource_id): + """ + Return the data dictionary info for a resource + """ + try: + return [ + f for f in get_action('datastore_search')( + None, {u'resource_id': resource_id, u'limit': 0})['fields'] + if not f['id'].startswith(u'_')] + except (ObjectNotFound, NotAuthorized): + return [] diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 067d268a9e4..2ab851c84a9 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -55,6 +55,7 @@ class DatastorePlugin(p.SingletonPlugin): p.implements(p.IDomainObjectModification, inherit=True) p.implements(p.IRoutes, inherit=True) p.implements(p.IResourceController, inherit=True) + p.implements(p.ITemplateHelpers) p.implements(interfaces.IDatastore, inherit=True) legacy_mode = False @@ -513,3 +514,7 @@ def _ts_rank_alias(self, field=None): if field: rank_alias += u' ' + field return u'"{0}"'.format(rank_alias) + + def get_helpers(self): + return { + 'datastore_dictionary': datastore_helpers.datastore_dictionary} diff --git a/ckanext/datastore/templates/package/resource_edit_base.html b/ckanext/datastore/templates/package/resource_edit_base.html index 1baa8e0b134..58332cd7b77 100644 --- a/ckanext/datastore/templates/package/resource_edit_base.html +++ b/ckanext/datastore/templates/package/resource_edit_base.html @@ -2,5 +2,7 @@ {% block inner_primary_nav %} {{ super() }} - {{ h.build_nav_icon('resource_dictionary', _('Data Dictionary'), id=pkg.name, resource_id=res.id) }} + {% if res.datastore_active %} + {{ h.build_nav_icon('resource_dictionary', _('Data Dictionary'), id=pkg.name, resource_id=res.id) }} + {% endif %} {% endblock %} diff --git a/ckanext/datastore/templates/package/resource_read.html b/ckanext/datastore/templates/package/resource_read.html index 636568e9cb4..1a17ae88947 100644 --- a/ckanext/datastore/templates/package/resource_read.html +++ b/ckanext/datastore/templates/package/resource_read.html @@ -2,5 +2,36 @@ {% block resource_actions_inner %} {{ super() }} -
  • {% snippet 'package/snippets/data_api_button.html', resource=res, datastore_root_url=c.datastore_api %}
    • -{% endblock %} \ No newline at end of file + {% if res.datastore_active %} +
  • {% snippet 'package/snippets/data_api_button.html', resource=res, datastore_root_url=c.datastore_api %}
    • + {% endif %} +{% endblock %} + +{% block resource_additional_information_inner %} + {% if res.datastore_active %} + {% set ddict=h.datastore_dictionary(res.id) %} +
    +

    {{ _('Data Dictionary') }}

    + + + + + + + + + + {% for f in ddict %} + + + + + + + {% endfor %} +
    {{ _('Column') }}{{ _('Type') }}{{ _('Label') }}{{ _('Description') }}
    {{ f.id }}{{ f.type }}{{ h.get_translated(f.get('info', {}), 'label') }}{{ h.render_markdown( + h.get_translated(f.get('info', {}), 'notes')) }}
    +
    + {% endif %} + {{ super() }} +{% endblock %} From 895b7ede841fdccd8165571a2f481beb8d98b16b Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Tue, 7 Feb 2017 14:24:51 -0500 Subject: [PATCH 10/25] allow package_search to return specified fields --- ckan/logic/action/get.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 32ac4cc968b..aa5ed714ed2 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -1831,10 +1831,9 @@ def package_search(context, data_dict): fl The parameter that controls which fields are returned in the solr - query cannot be changed. CKAN always returns the matched datasets as - dictionary objects. - if fl='id' or 'name', only this field in the dataset dictionary will - be returned. + query. + fl can be None or a list of result fields, such as ['id', 'extras_custom_field']. + if fl = None, datasets are returned as a list of full dictionary. ''' # sometimes context['schema'] is None schema = (context.get('schema') or @@ -1878,9 +1877,11 @@ def package_search(context, data_dict): data_source = 'validated_data_dict' data_dict.pop('use_default_schema', None) - # return a list of package ids - if data_dict.get('fl') not in ('id', 'name'): + result_fl = data_dict.get('fl') + if not result_fl: data_dict['fl'] = 'id {0}'.format(data_source) + else: + data_dict['fl'] = ' '.join(result_fl) # Remove before these hit solr FIXME: whitelist instead include_private = asbool(data_dict.pop('include_private', False)) @@ -1907,9 +1908,12 @@ def package_search(context, data_dict): # Add them back so extensions can use them on after_search data_dict['extras'] = extras - if data_dict.get('fl') in ['id', 'name']: + if result_fl: for package in query.results: - results.append( {data_dict.get('fl'):package} ) + if package.get('extras'): + package.update(package['extras'] ) + package.pop('extras') + results.append(package) else: for package in query.results: # get the package object From 3fdb4c171625ea6e6d32fc5132b43f63b24883e8 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Wed, 8 Feb 2017 08:47:55 -0500 Subject: [PATCH 11/25] add 'fl' in default_package_search_schema --- ckan/logic/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index 9f842d04b49..3a767f6d322 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -587,6 +587,7 @@ def default_autocomplete_schema(): def default_package_search_schema(): schema = { 'q': [ignore_missing, unicode], + 'fl': [ignore_missing, list_of_strings], 'fq': [ignore_missing, unicode], 'rows': [ignore_missing, natural_number_validator], 'sort': [ignore_missing, unicode], From 870aff94e4a058a690f86b6f9524376a304cac46 Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Wed, 8 Feb 2017 15:32:00 -0500 Subject: [PATCH 12/25] add test case for package_search() 'fl' support --- ckan/tests/logic/action/test_get.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index 8800c5b1980..a7bae7d3586 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -852,6 +852,17 @@ def test_search(self): eq(search_result['results'][0]['title'], 'Rivers') eq(search_result['count'], 1) + def test_search_fl(self): + factories.Dataset(title='Rivers', name='test_ri') + factories.Dataset(title='Lakes') + + search_result = helpers.call_action('package_search', q='rivers', fl=['title', 'name']) + + eq(search_result['results'][0]['title'], 'Rivers') + eq(search_result['results'][0]['name'], 'test_ri') + eq(len(search_result['results'][0]), 2) + eq(search_result['count'], 1) + def test_search_all(self): factories.Dataset(title='Rivers') factories.Dataset(title='Lakes') From bd03c7ba992e480c477b0981bb79dc103635f898 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 9 Feb 2017 09:56:02 -0500 Subject: [PATCH 13/25] [#3414] review clean-up --- .../templates/package/resource_edit_base.html | 2 +- ckanext/datastore/controller.py | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/ckanext/datapusher/templates/package/resource_edit_base.html b/ckanext/datapusher/templates/package/resource_edit_base.html index 9ae3dda1fdb..736c434c0b8 100644 --- a/ckanext/datapusher/templates/package/resource_edit_base.html +++ b/ckanext/datapusher/templates/package/resource_edit_base.html @@ -2,5 +2,5 @@ {% block inner_primary_nav %} {{ super() }} - {{ h.build_nav_icon('resource_data', _('DataPusher'), id=pkg.name, resource_id=res.id) }} + {{ h.build_nav_icon('resource_data', _('DataStore'), id=pkg.name, resource_id=res.id) }} {% endblock %} diff --git a/ckanext/datastore/controller.py b/ckanext/datastore/controller.py index 2267921e3e6..2d11e3d5adf 100644 --- a/ckanext/datastore/controller.py +++ b/ckanext/datastore/controller.py @@ -128,20 +128,3 @@ def dictionary(self, id, resource_id): return render( 'datastore/dictionary.html', extra_vars={'fields': fields}) - - -def short_hash(field): - u''' - return a short hash (20 hex digits) of a json-compatible object. - - We're using this to identify fields modified by the user when they - submit a form, and to detect if the source data changed before the - user submitted their changes. There's no need for a strong - cryptographic hash because this hash is only for preventing - accidental conflicts not enforcing access controls. - ''' - return md5.md5(json.dumps( - field, - sort_keys=True, - ensure_ascii=False, - separators=(u',', u':').encode('utf-8'))).hexdigest()[:20] From dc94a5e08f9bde3151795e38cb1164b7d8338ceb Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Fri, 10 Feb 2017 08:44:09 -0500 Subject: [PATCH 14/25] fix test case --- ckan/tests/logic/action/test_get.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index a7bae7d3586..c3f49f3b0db 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -858,10 +858,7 @@ def test_search_fl(self): search_result = helpers.call_action('package_search', q='rivers', fl=['title', 'name']) - eq(search_result['results'][0]['title'], 'Rivers') - eq(search_result['results'][0]['name'], 'test_ri') - eq(len(search_result['results'][0]), 2) - eq(search_result['count'], 1) + eq(search_result['results'], [{'title': 'Rivers', 'name': 'test_ri'}]) def test_search_all(self): factories.Dataset(title='Rivers') From 90338329bac9c5f10675f9526f69f61e8680c334 Mon Sep 17 00:00:00 2001 From: Gleb Date: Tue, 21 Feb 2017 07:13:56 +0200 Subject: [PATCH 15/25] #3259 / Organizations are sorted by title --- ckan/logic/action/get.py | 2 +- ckan/templates/organization/index.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index 9d96565b764..9336915ce8a 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -336,7 +336,7 @@ def _group_or_org_list(context, data_dict, is_org=False): data_dict, logic.schema.default_pagination_schema(), context) if errors: raise ValidationError(errors) - sort = data_dict.get('sort') or 'name' + sort = data_dict.get('sort') or 'title' q = data_dict.get('q') all_fields = asbool(data_dict.get('all_fields', None)) diff --git a/ckan/templates/organization/index.html b/ckan/templates/organization/index.html index d4079f318cf..4214ce6b165 100644 --- a/ckan/templates/organization/index.html +++ b/ckan/templates/organization/index.html @@ -17,7 +17,7 @@ {% block primary_content_inner %}

    {% block page_heading %}{{ _('Organizations') }}{% endblock %}

    {% block organizations_search_form %} - {% snippet 'snippets/search_form.html', form_id='organization-search-form', type='organization', query=c.q, sorting_selected=c.sort_by_selected, count=c.page.item_count, placeholder=_('Search organizations...'), show_empty=request.params, no_bottom_border=true if c.page.items %} + {% snippet 'snippets/search_form.html', form_id='organization-search-form', type='organization', query=c.q, sorting_selected=c.sort_by_selected, count=c.page.item_count, placeholder=_('Search organizations...'), show_empty=request.params, no_bottom_border=true if c.page.items, sorting = [(_('Name Ascending'), 'title asc'), (_('Name Descending'), 'title desc')] %} {% endblock %} {% block organizations_list %} {% if c.page.items or request.params %} From 1c2a0571b4ea34c728c0eb5a66545b0d4cf54ba9 Mon Sep 17 00:00:00 2001 From: Artem Bazykin Date: Tue, 21 Feb 2017 15:35:33 +0200 Subject: [PATCH 16/25] [#2651] Fix for delete resource on draft dataset should redirect to new_resource page --- ckan/controllers/package.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index aa7d7efeafd..7b92ab101b2 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -1041,7 +1041,12 @@ def resource_delete(self, id, resource_id): if request.method == 'POST': get_action('resource_delete')(context, {'id': resource_id}) h.flash_notice(_('Resource has been deleted.')) - h.redirect_to(controller='package', action='read', id=id) + pkg_dict = get_action('package_show')(context, {'id': id}) + if pkg_dict['state'].startswith('draft'): + h.redirect_to(controller='package', action='new_resource', + id=id) + else: + h.redirect_to(controller='package', action='read', id=id) c.resource_dict = get_action('resource_show')( context, {'id': resource_id}) c.pkg_id = id From 47b0122379eb8ee7a3659f78f76c7079e5b3629f Mon Sep 17 00:00:00 2001 From: Artem Bazykin Date: Thu, 23 Feb 2017 09:20:56 +0200 Subject: [PATCH 17/25] Use None in package_show instead context --- ckan/controllers/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 7b92ab101b2..837adbe6762 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -1041,7 +1041,7 @@ def resource_delete(self, id, resource_id): if request.method == 'POST': get_action('resource_delete')(context, {'id': resource_id}) h.flash_notice(_('Resource has been deleted.')) - pkg_dict = get_action('package_show')(context, {'id': id}) + pkg_dict = get_action('package_show')(None, {'id': id}) if pkg_dict['state'].startswith('draft'): h.redirect_to(controller='package', action='new_resource', id=id) From 363619b5aa17ec1b4b9f5a9d0033bc82b1eb23ee Mon Sep 17 00:00:00 2001 From: Artem Bazykin Date: Fri, 24 Feb 2017 10:43:26 +0200 Subject: [PATCH 18/25] Fix tags on org/group read pages --- ckan/controllers/group.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index a1480a5fda0..e711fea8561 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -270,8 +270,9 @@ def drill_down_url(**by): c.drill_down_url = drill_down_url def remove_field(key, value=None, replace=None): + controller = lookup_group_controller(group_type) return h.remove_url_param(key, value=value, replace=replace, - controller='group', action='read', + controller=controller, action='read', extras=dict(id=c.group_dict.get('name'))) c.remove_field = remove_field @@ -283,6 +284,7 @@ def pager_url(q=None, page=None): try: c.fields = [] + c.fields_grouped = {} search_extras = {} for (param, value) in request.params.items(): if param not in ['q', 'page', 'sort'] \ @@ -290,6 +292,10 @@ def pager_url(q=None, page=None): if not param.startswith('ext_'): c.fields.append((param, value)) q += ' %s: "%s"' % (param, value) + if param not in c.fields_grouped: + c.fields_grouped[param] = [value] + else: + c.fields_grouped[param].append(value) else: search_extras[param] = value From 72eb146bcdc2ee709ca7f2c02ec3f920961166e8 Mon Sep 17 00:00:00 2001 From: Tyler Kennedy Date: Sat, 4 Mar 2017 14:01:02 -0500 Subject: [PATCH 19/25] Performance improvements in group_activity_query --- ckan/model/activity.py | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/ckan/model/activity.py b/ckan/model/activity.py index fdfa52660cc..1a42a2023fd 100644 --- a/ckan/model/activity.py +++ b/ckan/model/activity.py @@ -3,7 +3,15 @@ import datetime from sqlalchemy import ( - orm, types, Column, Table, ForeignKey, desc, or_, union_all) + orm, + types, + Column, + Table, + ForeignKey, + desc, + or_, + union_all +) import ckan.model import meta @@ -182,14 +190,29 @@ def _group_activity_query(group_id): # Return a query with no results. return model.Session.query(model.Activity).filter("0=1") - dataset_ids = [dataset.id for dataset in group.packages()] + q = model.Session.query( + model.Activity + ).join( + model.Member, + model.Activity.object_id == model.Member.table_id + ).join( + model.Package, + model.Package.id == model.Member.table_id + ).filter( + # We only care about activity either on the the group itself or on + # packages within that group. + # FIXME: This means that activity that occured while a package belonged + # to a group but was then removed will not show up. This may not be + # desired. + or_( + model.Member.group_id == group_id, + model.Activity.object_id == group_id + ), + model.Member.state == 'active', + model.Member.table_name == 'package', + model.Package.private == False + ) - q = model.Session.query(model.Activity) - if dataset_ids: - q = q.filter(or_(model.Activity.object_id == group_id, - model.Activity.object_id.in_(dataset_ids))) - else: - q = q.filter(model.Activity.object_id == group_id) return q From 060307a42c6efe69d9766884ca9a41028c90ad23 Mon Sep 17 00:00:00 2001 From: Tyler Kennedy Date: Tue, 7 Mar 2017 02:10:08 -0500 Subject: [PATCH 20/25] Activity queries should return *all* activity events, not just those related to packages. --- ckan/model/activity.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/ckan/model/activity.py b/ckan/model/activity.py index 1a42a2023fd..2c3717dc902 100644 --- a/ckan/model/activity.py +++ b/ckan/model/activity.py @@ -10,6 +10,7 @@ ForeignKey, desc, or_, + and_, union_all ) @@ -192,25 +193,29 @@ def _group_activity_query(group_id): q = model.Session.query( model.Activity - ).join( + ).outerjoin( model.Member, - model.Activity.object_id == model.Member.table_id - ).join( + and_( + model.Activity.object_id == model.Member.table_id, + model.Member.state == 'active' + ) + ).outerjoin( model.Package, - model.Package.id == model.Member.table_id + and_( + model.Package.id == model.Member.table_id, + model.Package.private == False, + model.Package.state == 'active' + ) ).filter( # We only care about activity either on the the group itself or on # packages within that group. # FIXME: This means that activity that occured while a package belonged # to a group but was then removed will not show up. This may not be - # desired. + # desired but is consistent with legacy behaviour. or_( model.Member.group_id == group_id, model.Activity.object_id == group_id ), - model.Member.state == 'active', - model.Member.table_name == 'package', - model.Package.private == False ) return q From 4fb48e52fc2e1fa51f6ba4bebc72e0f86936028d Mon Sep 17 00:00:00 2001 From: Jinfei Fan Date: Mon, 13 Mar 2017 14:30:32 -0400 Subject: [PATCH 21/25] fix edit resource of draft dataset --- ckan/controllers/package.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index aa7d7efeafd..d57dee05e00 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -579,12 +579,7 @@ def resource_edit(self, id, resource_id, data=None, errors=None, # dataset has not yet been fully created resource_dict = get_action('resource_show')(context, {'id': resource_id}) - fields = ['url', 'resource_type', 'format', 'name', 'description', - 'id'] - data = {} - for field in fields: - data[field] = resource_dict[field] - return self.new_resource(id, data=data) + return self.new_resource(id, data=resource_dict) # resource is fully created try: resource_dict = get_action('resource_show')(context, From 81d9e44b71f9ed6ec607cb0eb966265a2c524b90 Mon Sep 17 00:00:00 2001 From: Jana Sloukova Date: Tue, 14 Mar 2017 17:15:26 +0100 Subject: [PATCH 22/25] Setting of datastore_active flag moved to separate function --- ckanext/datastore/logic/action.py | 95 ++++++++++++++++--------------- 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index e519670bf19..8d45f837b91 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -149,47 +149,7 @@ def datastore_create(context, data_dict): log.debug( 'Setting datastore_active=True on resource {0}'.format(resource.id) ) - # issue #3245: race condition - update_dict = {'datastore_active': True} - - # get extras(for entity update) and package_id(for search index update) - res_query = model.Session.query( - model.resource_table.c.extras, - model.resource_table.c.package_id - ).filter( - model.Resource.id == data_dict['resource_id'] - ) - extras, package_id = res_query.one() - - # update extras in database for record and its revision - extras.update(update_dict) - res_query.update({'extras': extras}, synchronize_session=False) - - model.Session.query(model.resource_revision_table).filter( - model.ResourceRevision.id == data_dict['resource_id'], - model.ResourceRevision.current is True - ).update({'extras': extras}, synchronize_session=False) - - model.Session.commit() - - # get package with updated resource from solr - # find changed resource, patch it and reindex package - psi = search.PackageSearchIndex() - solr_query = search.PackageSearchQuery() - q = { - 'q': 'id:"{0}"'.format(package_id), - 'fl': 'data_dict', - 'wt': 'json', - 'fq': 'site_id:"%s"' % config.get('ckan.site_id'), - 'rows': 1 - } - for record in solr_query.run(q)['results']: - solr_data_dict = json.loads(record['data_dict']) - for resource in solr_data_dict['resources']: - if resource['id'] == data_dict['resource_id']: - resource.update(update_dict) - psi.index_package(solr_data_dict) - break + set_datastore_active_flag(model, data_dict, True) result.pop('id', None) result.pop('private', None) @@ -396,11 +356,9 @@ def datastore_delete(context, data_dict): if (not data_dict.get('filters') and resource.extras.get('datastore_active') is True): log.debug( - 'Setting datastore_active=True on resource {0}'.format(resource.id) + 'Setting datastore_active=False on resource {0}'.format(resource.id) ) - p.toolkit.get_action('resource_patch')( - context, {'id': data_dict['resource_id'], - 'datastore_active': False}) + set_datastore_active_flag(model, data_dict, False) result.pop('id', None) result.pop('connection_url') @@ -598,6 +556,53 @@ def datastore_make_public(context, data_dict): db.make_public(context, data_dict) +def set_datastore_active_flag(model, data_dict, flag): + ''' + Set appropriate datastore_active flag on CKAN resource. + + Called after creation or deletion of DataStore table. + ''' + update_dict = {'datastore_active': flag} + + # get extras(for entity update) and package_id(for search index update) + res_query = model.Session.query( + model.resource_table.c.extras, + model.resource_table.c.package_id + ).filter( + model.Resource.id == data_dict['resource_id'] + ) + extras, package_id = res_query.one() + + # update extras in database for record and its revision + extras.update(update_dict) + res_query.update({'extras': extras}, synchronize_session=False) + model.Session.query(model.resource_revision_table).filter( + model.ResourceRevision.id == data_dict['resource_id'], + model.ResourceRevision.current is True + ).update({'extras': extras}, synchronize_session=False) + + model.Session.commit() + + # get package with updated resource from solr + # find changed resource, patch it and reindex package + psi = search.PackageSearchIndex() + solr_query = search.PackageSearchQuery() + q = { + 'q': 'id:"{0}"'.format(package_id), + 'fl': 'data_dict', + 'wt': 'json', + 'fq': 'site_id:"%s"' % config.get('ckan.site_id'), + 'rows': 1 + } + for record in solr_query.run(q)['results']: + solr_data_dict = json.loads(record['data_dict']) + for resource in solr_data_dict['resources']: + if resource['id'] == data_dict['resource_id']: + resource.update(update_dict) + psi.index_package(solr_data_dict) + break + + def _resource_exists(context, data_dict): ''' Returns true if the resource exists in CKAN and in the datastore ''' model = _get_or_bust(context, 'model') From 99cdca143f1b99eece7269fe20d6b478df96d5f7 Mon Sep 17 00:00:00 2001 From: Jari Voutilainen Date: Wed, 15 Mar 2017 18:03:20 +0200 Subject: [PATCH 23/25] Use h.url_for and qualified=True for reset mails --- ckan/lib/mailer.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py index 9eaf2fd6813..c75e3435e96 100644 --- a/ckan/lib/mailer.py +++ b/ckan/lib/mailer.py @@ -144,11 +144,11 @@ def get_invite_body(user, group_dict=None, role=None): def get_reset_link(user): - return urljoin(config.get('site_url'), - h.url_for(controller='user', - action='perform_reset', - id=user.id, - key=user.reset_key)) + return h.url_for(controller='user', + action='perform_reset', + id=user.id, + key=user.reset_key, + qualified=True) def send_reset_link(user): From 0e35ecf1e267c62faea85d46b6190a281a378557 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Thu, 16 Mar 2017 10:39:08 -0400 Subject: [PATCH 24/25] [#3481] comment explaining approach --- ckanext/datastore/logic/action.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ckanext/datastore/logic/action.py b/ckanext/datastore/logic/action.py index 8d45f837b91..27e74599668 100644 --- a/ckanext/datastore/logic/action.py +++ b/ckanext/datastore/logic/action.py @@ -562,6 +562,9 @@ def set_datastore_active_flag(model, data_dict, flag): Called after creation or deletion of DataStore table. ''' + # We're modifying the resource extra directly here to avoid a + # race condition, see issue #3245 for details and plan for a + # better fix update_dict = {'datastore_active': flag} # get extras(for entity update) and package_id(for search index update) From a8792a07e179a2557992f09efd1203fdd13c6db2 Mon Sep 17 00:00:00 2001 From: John Date: Tue, 21 Mar 2017 11:29:56 +0000 Subject: [PATCH 25/25] Fixed link for Jetty. Old Jetty link (http://jetty.codehaus.org/jetty/) was non-responsive. Replaced with link to the Eclipse Jetty documentation. --- doc/maintaining/installing/install-from-source.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index 71c3b23f5c3..c022946a9e7 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -40,7 +40,7 @@ pip `A tool for installing and managing Python packages `_ Git `A distributed version control system `_ Apache Solr `A search platform `_ -Jetty `An HTTP server `_ (used for Solr). +Jetty `An HTTP server `_ (used for Solr). OpenJDK 6 JDK `The Java Development Kit `_ Redis `An in-memory data structure store `_ ===================== ===============================================