From 8139eb8c7d154a9bd73443ca1da075d6c741e74d Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 11:09:43 +0000 Subject: [PATCH 01/13] Flask upgraded to 0.12.2 --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index c86f1288d17..909e0f658e4 100644 --- a/requirements.in +++ b/requirements.in @@ -5,7 +5,7 @@ Beaker==1.8.1 # Needs to be pinned to a more up to date version than the Pylons bleach==2.1.2 click==6.7 fanstatic==0.12 -Flask==0.11.1 +Flask==0.12.2 Flask-Babel==0.11.2 Jinja2==2.8 Markdown==2.6.7 diff --git a/requirements.txt b/requirements.txt index 468254a7d6f..3af73fe89cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,8 @@ bleach==2.1.2 click==6.7 decorator==4.0.6 # via pylons, sqlalchemy-migrate fanstatic==0.12 -Flask==0.11.1 Flask-Babel==0.11.2 +Flask==0.12.2 FormEncode==1.3.0 # via pylons funcsigs==1.0.2 # via beaker html5lib==1.0.1 # via bleach From 95264ebe9c57fd31463f3fa828521854a939c89d Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 15:01:45 +0000 Subject: [PATCH 02/13] pip-compile -U puts the latest version of Beaker in requirements.txt, so it was not useful to pin it in requirements.in --- requirements.in | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.in b/requirements.in index 97a2353ccbe..c72e16c082c 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,6 @@ # The file contains the direct ckan requirements. # Use pip-compile to create a requirements.txt file from this Babel==2.3.4 -Beaker==1.9.0 # Needs to be pinned to a more up to date version than the Pylons one bleach==2.1.2 click==6.7 fanstatic==0.12 From 7003fe8a4041632455ed72454cf80720485e2e0a Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 15:10:34 +0000 Subject: [PATCH 03/13] Updated deps by running: pip-compile -U --output-file requirements.txt requirements.in --- requirements.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9eedbfe53a9..2b05f76817c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,22 +5,22 @@ # pip-compile --output-file requirements.txt requirements.in # argparse==1.4.0 # via ofs -Babel==2.3.4 +Babel==2.3.4 # via flask-babel Beaker==1.9.0 # via pylons bleach==2.1.2 click==6.7 -decorator==4.0.6 # via pylons, sqlalchemy-migrate +decorator==4.2.1 # via pylons, sqlalchemy-migrate fanstatic==0.12 -Flask==0.11.1 Flask-Babel==0.11.2 -FormEncode==1.3.0 # via pylons +Flask==0.11.1 # via flask-babel +FormEncode==1.3.1 # via pylons funcsigs==1.0.2 # via beaker html5lib==1.0.1 # via bleach itsdangerous==0.24 # via flask -Jinja2==2.8 # via flask -Mako==1.0.4 # via pylons +Jinja2==2.8 # via flask, flask-babel +Mako==1.0.7 # via pylons Markdown==2.6.7 -MarkupSafe==0.23 # via jinja2, mako, webhelpers +MarkupSafe==1.0 # via jinja2, mako, webhelpers nose==1.3.7 # via pylons ofs==0.4.2 Pairtree==0.7.1-T @@ -31,22 +31,22 @@ PasteScript==2.0.2 # via pylons pbr==1.10.0 # via sqlalchemy-migrate polib==1.0.7 psycopg2==2.7.3.2 -Pygments==2.1.3 # via weberror +Pygments==2.2.0 # via weberror Pylons==0.9.7 pysolr==3.6.0 python-dateutil==1.5 python-magic==0.4.12 pytz==2016.7 pyutilib.component.core==4.6.4 -redis==2.10.5 # via rq -repoze.lru==0.6 # via routes +redis==2.10.6 # via rq +repoze.lru==0.7 # via routes repoze.who-friendlyform==1.0.8 repoze.who==2.3 requests==2.11.1 Routes==1.13 # via pylons rq==0.6.0 simplejson==3.10.0 -six==1.10.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate +six==1.11.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate sqlalchemy-migrate==0.10.0 SQLAlchemy==1.1.11 # via sqlalchemy-migrate sqlparse==0.2.2 @@ -59,7 +59,7 @@ WebError==0.13.1 # via pylons WebHelpers==1.3 # via pylons WebOb==1.0.8 # via fanstatic, pylons, repoze.who, repoze.who-friendlyform, weberror, webtest WebTest==1.4.3 # via pylons -Werkzeug==0.11.10 # via flask +Werkzeug==0.14.1 # via flask zope.interface==4.3.2 # The following packages are commented out because they are From e0f1aafa8ecd52dc6e8238826ca1e40dd92a266d Mon Sep 17 00:00:00 2001 From: cclauss Date: Sat, 17 Feb 2018 13:17:55 +0100 Subject: [PATCH 04/13] Change basestring --> six.string_types for Python 3 --- ckan/controllers/group.py | 3 ++- ckan/controllers/package.py | 3 ++- ckan/lib/dictization/model_save.py | 3 ++- ckan/lib/extract.py | 3 ++- ckan/lib/helpers.py | 15 ++++++++------- ckan/lib/search/common.py | 4 +++- ckan/lib/search/query.py | 8 ++++---- ckan/logic/__init__.py | 5 +++-- ckan/logic/action/get.py | 13 +++++++------ ckan/logic/converters.py | 10 ++++++---- ckan/model/types.py | 5 +++-- ckan/plugins/core.py | 3 ++- ckan/tests/legacy/html_check.py | 10 ++++++---- ckanext/datastore/backend/postgres.py | 10 ++++++---- ckanext/datastore/helpers.py | 10 ++++++---- ckanext/datastore/logic/schema.py | 6 ++++-- ckanext/datastore/plugin.py | 10 ++++++---- ckanext/multilingual/plugin.py | 12 +++++++----- 18 files changed, 79 insertions(+), 54 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index c70e4b612c9..968dce61bf4 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -5,6 +5,7 @@ from urllib import urlencode from pylons.i18n import get_lang +from six import string_types import ckan.lib.base as base import ckan.lib.helpers as h @@ -268,7 +269,7 @@ def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) - params = [(k, v.encode('utf-8') if isinstance(v, basestring) + params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 2c4a2e85914..fbb313a82f9 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -9,6 +9,7 @@ from ckan.common import config from paste.deploy.converters import asbool import paste.fileapp +from six import string_types import ckan.logic as logic import ckan.lib.base as base @@ -45,7 +46,7 @@ def _encode_params(params): - return [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) + return [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index 60476666682..790bfe23c95 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -5,6 +5,7 @@ import logging from sqlalchemy.orm import class_mapper +from six import string_types import ckan.lib.dictization as d import ckan.lib.helpers as h @@ -460,7 +461,7 @@ def package_api_to_dict(api1_dict, context): for key, value in api1_dict.iteritems(): new_value = value if key == 'tags': - if isinstance(value, basestring): + if isinstance(value, string_types): new_value = [{"name": item} for item in value.split()] else: new_value = [{"name": item} for item in value] diff --git a/ckan/lib/extract.py b/ckan/lib/extract.py index 927e53f5ca2..6a57ce664ac 100644 --- a/ckan/lib/extract.py +++ b/ckan/lib/extract.py @@ -3,6 +3,7 @@ import re from jinja2.ext import babel_extract as extract_jinja2 import lib.jinja_extensions +from six import string_types jinja_extensions = ''' jinja2.ext.do, jinja2.ext.with_, @@ -24,7 +25,7 @@ def jinja2_cleaner(fileobj, *args, **kw): for lineno, func, message, finder in raw_extract: - if isinstance(message, basestring): + if isinstance(message, string_types): message = lib.jinja_extensions.regularise_html(message) elif message is not None: message = (lib.jinja_extensions.regularise_html(message[0]) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 46f98164da7..523cc95547d 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -35,6 +35,7 @@ from flask import url_for as _flask_default_url_for from werkzeug.routing import BuildError as FlaskRouteBuildError import i18n +from six import string_types import ckan.exceptions import ckan.model as model @@ -111,7 +112,7 @@ def _datestamp_to_datetime(datetime_): :rtype: datetime ''' - if isinstance(datetime_, basestring): + if isinstance(datetime_, string_types): try: datetime_ = date_str_to_datetime(datetime_) except TypeError: @@ -171,7 +172,7 @@ def redirect_to(*args, **kw): _url = '' skip_url_parsing = False parse_url = kw.pop('parse_url', False) - if uargs and len(uargs) is 1 and isinstance(uargs[0], basestring) \ + if uargs and len(uargs) is 1 and isinstance(uargs[0], string_types) \ and (uargs[0].startswith('/') or is_url(uargs[0])) \ and parse_url is False: skip_url_parsing = True @@ -358,7 +359,7 @@ def _url_for_flask(*args, **kw): # The API routes used to require a slash on the version number, make sure # we remove it if (args and args[0].startswith('api.') and - isinstance(kw.get('ver'), basestring) and + isinstance(kw.get('ver'), string_types) and kw['ver'].startswith('/')): kw['ver'] = kw['ver'].replace('/', '') @@ -1051,7 +1052,7 @@ def get_param_int(name, default=10): def _url_with_params(url, params): if not params: return url - params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) + params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) @@ -1802,7 +1803,7 @@ def remove_url_param(key, value=None, replace=None, controller=None, instead. ''' - if isinstance(key, basestring): + if isinstance(key, string_types): keys = [key] else: keys = key @@ -2137,7 +2138,7 @@ def format_resource_items(items): # Sometimes values that can't be converted to ints can sneak # into the db. In this case, just leave them as they are. pass - elif isinstance(value, basestring): + elif isinstance(value, string_types): # check if strings are actually datetime/number etc if re.search(reg_ex_datetime, value): datetime_ = date_str_to_datetime(value) @@ -2534,7 +2535,7 @@ def get_translated(data_dict, field): return data_dict[field + u'_translated'][language] except KeyError: val = data_dict.get(field, '') - return _(val) if val and isinstance(val, basestring) else val + return _(val) if val and isinstance(val, string_types) else val @core_helper diff --git a/ckan/lib/search/common.py b/ckan/lib/search/common.py index 8d031891760..060f62f4175 100644 --- a/ckan/lib/search/common.py +++ b/ckan/lib/search/common.py @@ -5,6 +5,8 @@ import re import pysolr import simplejson +from six import string_types + log = logging.getLogger(__name__) @@ -72,7 +74,7 @@ def make_connection(decode_dates=True): def solr_datetime_decoder(d): for k, v in d.items(): - if isinstance(v, basestring): + if isinstance(v, string_types): possible_datetime = re.search(pysolr.DATETIME_REGEX, v) if possible_datetime: date_values = possible_datetime.groupdict() diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 13d75e7a799..9ad3adf0b1c 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -49,7 +49,7 @@ def convert_legacy_parameters_to_solr(legacy_params): non_solr_params = set(legacy_params.keys()) - VALID_SOLR_PARAMETERS for search_key in non_solr_params: value_obj = legacy_params[search_key] - value = value_obj.replace('+', ' ') if isinstance(value_obj, basestring) else value_obj + value = value_obj.replace('+', ' ') if isinstance(value_obj, six.string_types) else value_obj if search_key == 'all_fields': if value: solr_params['fl'] = '*' @@ -62,7 +62,7 @@ def convert_legacy_parameters_to_solr(legacy_params): elif search_key == 'tags': if isinstance(value_obj, list): tag_list = value_obj - elif isinstance(value_obj, basestring): + elif isinstance(value_obj, six.string_types): tag_list = [value_obj] else: raise SearchQueryError('Was expecting either a string or JSON list for the tags parameter: %r' % value) @@ -173,7 +173,7 @@ def run(self, query=None, fields=None, options=None, **kwargs): else: options.update(kwargs) - if isinstance(query, basestring): + if isinstance(query, six.string_types): query = [query] query = query[:] # don't alter caller's query list. @@ -220,7 +220,7 @@ def run(self, fields={}, options=None, **kwargs): # action. query = [] for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, six.string_types): terms = terms.split() for term in terms: query.append(':'.join([field, term])) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index e5dee92206e..fdbf527806e 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -7,6 +7,7 @@ from collections import defaultdict import formencode.validators +from six import string_types import ckan.model as model import ckan.authz as authz @@ -175,7 +176,7 @@ def clean_dict(data_dict): if not isinstance(value, list): continue for inner_dict in value[:]: - if isinstance(inner_dict, basestring): + if isinstance(inner_dict, string_types): break if not any(inner_dict.values()): value.remove(inner_dict) @@ -516,7 +517,7 @@ def get_or_bust(data_dict, keys): not in the given dictionary ''' - if isinstance(keys, basestring): + if isinstance(keys, string_types): keys = [keys] import ckan.logic.schema as schema diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index d68d3b310d2..a2c9e0fcbf9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -11,6 +11,7 @@ from ckan.common import config import sqlalchemy from paste.deploy.converters import asbool +from six import string_types import ckan.lib.dictization import ckan.logic as logic @@ -2109,7 +2110,7 @@ def resource_search(context, data_dict): {'fields': _('Do not specify if using "query" parameter')}) elif query is not None: - if isinstance(query, basestring): + if isinstance(query, string_types): query = [query] try: fields = dict(pair.split(":", 1) for pair in query) @@ -2125,7 +2126,7 @@ def resource_search(context, data_dict): # So maintain that behaviour split_terms = {} for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = terms.split() split_terms[field] = terms fields = split_terms @@ -2143,7 +2144,7 @@ def resource_search(context, data_dict): resource_fields = model.Resource.get_columns() for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] if field not in resource_fields: @@ -2215,7 +2216,7 @@ def _tag_search(context, data_dict): model = context['model'] terms = data_dict.get('query') or data_dict.get('q') or [] - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] terms = [t.strip() for t in terms if t.strip()] @@ -2402,7 +2403,7 @@ def term_translation_show(context, data_dict): # This action accepts `terms` as either a list of strings, or a single # string. terms = _get_or_bust(data_dict, 'terms') - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] if terms: q = q.where(trans_table.c.term.in_(terms)) @@ -2411,7 +2412,7 @@ def term_translation_show(context, data_dict): # string. if 'lang_codes' in data_dict: lang_codes = _get_or_bust(data_dict, 'lang_codes') - if isinstance(lang_codes, basestring): + if isinstance(lang_codes, string_types): lang_codes = [lang_codes] q = q.where(trans_table.c.lang_code.in_(lang_codes)) diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py index c747555e910..ed97f67de73 100644 --- a/ckan/logic/converters.py +++ b/ckan/logic/converters.py @@ -2,6 +2,8 @@ import json +from six import string_types + import ckan.model as model import ckan.lib.navl.dictization_functions as df import ckan.logic.validators as validators @@ -60,7 +62,7 @@ def callable(key, data, errors, context): new_tags = data.get(key) if not new_tags: return - if isinstance(new_tags, basestring): + if isinstance(new_tags, string_types): new_tags = [new_tags] # get current number of tags @@ -173,7 +175,7 @@ def convert_group_name_or_id_to_id(group_name_or_id, context): def convert_to_json_if_string(value, context): - if isinstance(value, basestring): + if isinstance(value, string_types): try: return json.loads(value) except ValueError: @@ -183,13 +185,13 @@ def convert_to_json_if_string(value, context): def convert_to_list_if_string(value, context=None): - if isinstance(value, basestring): + if isinstance(value, string_types): return [value] else: return value def remove_whitespace(value, context): - if isinstance(value, basestring): + if isinstance(value, string_types): return value.strip() return value diff --git a/ckan/model/types.py b/ckan/model/types.py index b05fcf335eb..c4351730126 100644 --- a/ckan/model/types.py +++ b/ckan/model/types.py @@ -7,6 +7,7 @@ import simplejson as json from sqlalchemy import types +from six import string_types import meta @@ -74,7 +75,7 @@ def process_bind_param(self, value, engine): if value is None or value == {}: # ensure we stores nulls in db not json "null" return None else: - if isinstance(value, basestring): + if isinstance(value, string_types): return unicode(value) else: return unicode(json.dumps(value, ensure_ascii=False)) @@ -89,7 +90,7 @@ def iso_date_to_datetime_for_sqlite(datetime_or_iso_date_if_sqlite): # to call this to convert it into a datetime type. When running on # postgres then you have a datetime anyway, so this function doesn't # do anything. - if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, basestring): + if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, string_types): return datetime.datetime.strptime(datetime_or_iso_date_if_sqlite, '%Y-%m-%d %H:%M:%S.%f') else: diff --git a/ckan/plugins/core.py b/ckan/plugins/core.py index 07456fc8803..14098016d0d 100644 --- a/ckan/plugins/core.py +++ b/ckan/plugins/core.py @@ -12,6 +12,7 @@ from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin from pyutilib.component.core import Plugin as _pca_Plugin from paste.deploy.converters import asbool +from six import string_types import interfaces @@ -244,7 +245,7 @@ def _get_service(plugin_name): :return: the service object ''' - if isinstance(plugin_name, basestring): + if isinstance(plugin_name, string_types): for group in GROUPS: iterator = iter_entry_points( group=group, diff --git a/ckan/tests/legacy/html_check.py b/ckan/tests/legacy/html_check.py index 5b549942ee6..e8bc05e502b 100644 --- a/ckan/tests/legacy/html_check.py +++ b/ckan/tests/legacy/html_check.py @@ -3,13 +3,15 @@ import re import sgmllib +from six import string_types + import paste.fixture class HtmlCheckMethods(object): '''A collection of methods to check properties of a html page, usually in the form returned by paster.''' - + def named_div(self, div_name, html): 'strips html to just the
section' the_html = self._get_html_from_res(html) @@ -31,15 +33,15 @@ def sidebar(self, html): def strip_tags(self, res): '''Call strip_tags on a TestResponse object to strip any and all HTML and normalise whitespace.''' - if not isinstance(res, basestring): + if not isinstance(res, string_types): res = res.body.decode('utf-8') - return Stripper().strip(res) + return Stripper().strip(res) def check_named_element(self, html, tag_name, *html_to_find): '''Searches in the html and returns True if it can find a particular tag and all its subtags & data which contains all the of the html_to_find''' - named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?)' % {'tag':tag_name}) + named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?)' % {'tag':tag_name}) html_str = self._get_html_from_res(html) self._check_html(named_element_re, html_str.replace('\n', ''), html_to_find) diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index c0c287a9c21..7c4118dc5b8 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -15,6 +15,8 @@ import json from cStringIO import StringIO +from six import string_types + import ckan.lib.cli as cli import ckan.plugins as p import ckan.plugins.toolkit as toolkit @@ -374,7 +376,7 @@ def _where_clauses(data_dict, fields_types): # add full-text search where clause q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): ts_query_alias = _ts_query_alias() clause_str = u'_full_text @@ {0}'.format(ts_query_alias) clauses.append((clause_str,)) @@ -409,7 +411,7 @@ def _textsearch_query(data_dict): statements = [] rank_columns = [] plain = data_dict.get('plain', True) - if isinstance(q, basestring): + if isinstance(q, string_types): query, rank = _build_query_and_rank_statements( lang, q, plain) statements.append(query) @@ -469,7 +471,7 @@ def _sort(data_dict, fields_types): if not sort: q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): return [_ts_rank_alias()] elif isinstance(q, dict): return [_ts_rank_alias(field) for field in q @@ -1196,7 +1198,7 @@ def validate(context, data_dict): for key, values in data_dict_copy.iteritems(): if not values: continue - if isinstance(values, basestring): + if isinstance(values, string_types): value = values elif isinstance(values, (list, tuple)): value = values[0] diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py index b616f0f9405..523bb096774 100644 --- a/ckanext/datastore/helpers.py +++ b/ckanext/datastore/helpers.py @@ -6,6 +6,8 @@ import paste.deploy.converters as converters import sqlparse +from six import string_types + from ckan.plugins.toolkit import get_action, ObjectNotFound, NotAuthorized log = logging.getLogger(__name__) @@ -57,10 +59,10 @@ def validate_int(i, non_negative=False): return i >= 0 or not non_negative -def _strip(input): - if isinstance(input, basestring) and len(input) and input[0] == input[-1]: - return input.strip().strip('"') - return input +def _strip(s): + if isinstance(s, string_types) and len(s) and s[0] == s[-1]: + return s.strip().strip('"') + return s def should_fts_index_field_type(field_type): diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 903e016d71c..9a7d5795925 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -2,6 +2,8 @@ import json +from six import string_types + import ckan.plugins as p import ckan.lib.navl.dictization_functions as df @@ -48,13 +50,13 @@ def list_of_strings_or_lists(key, data, errors, context): if not isinstance(value, list): raise df.Invalid('Not a list') for x in value: - if not isinstance(x, basestring) and not isinstance(x, list): + if not isinstance(x, string_types) and not isinstance(x, list): raise df.Invalid('%s: %s' % ('Neither a string nor a list', x)) def list_of_strings_or_string(key, data, errors, context): value = data.get(key) - if isinstance(value, basestring): + if isinstance(value, string_types): return list_of_strings_or_lists(key, data, errors, context) diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index f47cbbfd796..8d0c6b7b584 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -2,6 +2,7 @@ import logging +from six import string_types import ckan.plugins as p import ckan.logic as logic @@ -213,16 +214,17 @@ def datastore_validate(self, context, data_dict, fields_types): q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): del data_dict['q'] elif isinstance(q, dict): for key in q.keys(): - if key in fields_types and isinstance(q[key], basestring): + if key in fields_types and isinstance(q[key], + string_types): del q[key] language = data_dict.get('language') if language: - if isinstance(language, basestring): + if isinstance(language, string_types): del data_dict['language'] plain = data_dict.get('plain') @@ -249,7 +251,7 @@ def datastore_validate(self, context, data_dict, fields_types): if limit: is_positive_int = datastore_helpers.validate_int(limit, non_negative=True) - is_all = isinstance(limit, basestring) and limit.lower() == 'all' + is_all = isinstance(limit, string_types) and limit.lower() == 'all' if is_positive_int or is_all: del data_dict['limit'] diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py index 238aa5e7371..c4615151572 100644 --- a/ckanext/multilingual/plugin.py +++ b/ckanext/multilingual/plugin.py @@ -1,5 +1,7 @@ # encoding: utf-8 +from six import string_types + import ckan from ckan.plugins import SingletonPlugin, implements, IPackageController from ckan.plugins import IGroupController, IOrganizationController, ITagController, IResourceController @@ -31,7 +33,7 @@ def translate_data_dict(data_dict): for (key, value) in flattened.items(): if value in (None, True, False): continue - elif isinstance(value, basestring): + elif isinstance(value, string_types): terms.add(value) elif isinstance(value, (int, long)): continue @@ -79,7 +81,7 @@ def translate_data_dict(data_dict): # Don't try to translate values that aren't strings. translated_flattened[key] = value - elif isinstance(value, basestring): + elif isinstance(value, string_types): if value in desired_translations: translated_flattened[key] = desired_translations[value] else: @@ -127,7 +129,7 @@ def translate_resource_data_dict(data_dict): for (key, value) in flattened.items(): if value in (None, True, False): continue - elif isinstance(value, basestring): + elif isinstance(value, string_types): terms.add(value) elif isinstance(value, (int, long)): continue @@ -170,7 +172,7 @@ def translate_resource_data_dict(data_dict): # Don't try to translate values that aren't strings. translated_flattened[key] = value - elif isinstance(value, basestring): + elif isinstance(value, string_types): if value in desired_translations: translated_flattened[key] = desired_translations[value] else: @@ -229,7 +231,7 @@ def before_index(self, search_data): if not isinstance(value, list): value = [value] for item in value: - if isinstance(item, basestring): + if isinstance(item, string_types): all_terms.append(item) field_translations = get_action('term_translation_show')( From ec795ab4d20078247e35216dda7d7840907f27f6 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 17:42:24 -0500 Subject: [PATCH 05/13] [#3989] inline docs in dictionary_form.html --- .../datastore/snippets/dictionary_form.html | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html index fad1d662df7..41266c9aae9 100644 --- a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html +++ b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html @@ -1,16 +1,25 @@ {% import 'macros/form.html' as form %}

{{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})

- {{ form.select('info__' ~ position ~ '__type_override', - label=_('Type Override'), options=[ - {'name': '', 'value': ''}, - {'name': 'text', 'value': 'text'}, - {'name': 'numeric', 'value': 'numeric'}, - {'name': 'timestamp', 'value': 'timestamp'}, - ], selected=field.get('info', {}).get('type_override', '')) }} - {{ form.input('info__' ~ position ~ '__label', - label=_('Label'), id='field-f' ~ position ~ 'label', - value=field.get('info', {}).get('label', ''), classes=['control-full']) }} - {{ form.markdown('info__' ~ position ~ '__notes', - label=_('Description'), id='field-d' ~ position ~ 'notes', - value=field.get('info', {}).get('notes', '')) }} + +{# + Data Dictionary fields may be added this snippet. New fields following + the 'info__' ~ position ~ '__namegoeshere' convention will be saved + as part of the "info" object on the column. +#} + +{{ form.select('info__' ~ position ~ '__type_override', + label=_('Type Override'), options=[ + {'name': '', 'value': ''}, + {'name': 'text', 'value': 'text'}, + {'name': 'numeric', 'value': 'numeric'}, + {'name': 'timestamp', 'value': 'timestamp'}, + ], selected=field.get('info', {}).get('type_override', '')) }} + +{{ form.input('info__' ~ position ~ '__label', + label=_('Label'), id='field-f' ~ position ~ 'label', + value=field.get('info', {}).get('label', ''), classes=['control-full']) }} + +{{ form.markdown('info__' ~ position ~ '__notes', + label=_('Description'), id='field-d' ~ position ~ 'notes', + value=field.get('info', {}).get('notes', '')) }} From 699856611f1f82b4bc4f580abc8f85e0408f728e Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 19:33:39 -0500 Subject: [PATCH 06/13] [#3989] document field 'info' object --- doc/maintaining/datastore.rst | 42 ++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index c4da16b5d16..03d7627aaad 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -304,22 +304,40 @@ Fields Fields define the column names and the type of the data in a column. A field is defined as follows:: { - "id": # a string which defines the column name + "id": # the column name (required) "type": # the data type for the column + "info": { + "label": # human-readable label for column + "notes": # markdown description of column + "type_override": # type for datapusher to use when importing data + ...: # other user-defined fields + } } -Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid. +Field types not provided will be guessed based on the first row of provided data. +Set the types to ensure that future inserts will not fail because of an incorrectly +guessed type. See :ref:`valid-types` for details on which types are valid. + +Extra ``"info"`` field values will be stored along with the column. ``"label"``, +``"notes"`` and ``"type_override"`` can be managed from the default Data Dictionary +form. Additional fields can be stored by customizing the Data Dictionary form or by +passing their values to the API directly. Example:: [ { - "id": "foo", - "type": "int4" + "id": "code_number", + "type": "numeric" }, { - "id": "bar" - # type is optional + "id": "description" + "type": "text", + "info": { + "label": "Description", + "notes": "A brief usage description for this code", + "example": "Used for temporary service interruptions" + } } ] @@ -331,19 +349,21 @@ Records A record is the data to be inserted in a DataStore resource and is defined as follows:: { - "": # data to be set - # .. more data + column_1_id: value_1, + columd_2_id: value_2, + ... } Example:: [ { - "foo": 100, - "bar": "Here's some text" + "code_number": 10, + "description": "Submitted successfully" }, { - "foo": 42 + "code_number": 42, + "description": "In progress" } ] From eba5abbdc71f5c2d555c958507bbe5718246eb69 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 19:40:54 -0500 Subject: [PATCH 07/13] [#3989] promote "Download resource" and "Extending DataStore" to top level sections --- doc/maintaining/datastore.rst | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index 03d7627aaad..80bffd22bc3 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -233,6 +233,25 @@ alongside CKAN. To install this please look at the docs here: http://docs.ckan.org/projects/datapusher +.. _dump: + +--------------------- +Downloading resources +--------------------- + +A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. + +For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``. + +Other formats supported include tab-separated values (``?format=tsv``), +JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible +tab-separated file use +``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``. + +.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values + + + ----------------- The DataStore API ----------------- @@ -279,23 +298,6 @@ API reference :members: -.. _dump: - -Download resource ------------------ - -A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. - -For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``. - -Other formats supported include tab-separated values (``?format=tsv``), -JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible -tab-separated file use -``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``. - -.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values - - .. _fields: Fields @@ -457,8 +459,10 @@ name oid The PostgreSQL object ID of the table that belongs to name. + +------------------- Extending DataStore -=================== +------------------- Starting from CKAN version 2.7, backend used in DataStore can be replaced with custom one. For this purpose, custom extension must implement `ckanext.datastore.interfaces.IDatastoreBackend`, which provides one method - `register_backends`. It should return dictonary with names of custom backends as keys and classes, that represent those backends as values. Each class supposed to be inherited from `ckanext.datastore.backend.DatastoreBackend`. From 297e4295eb61d645d9a6e1d83de13846d447bc05 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 20:07:12 -0500 Subject: [PATCH 08/13] [#3989] document Data Dictionary form --- doc/maintaining/datastore.rst | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index 80bffd22bc3..0fe2db7de2b 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -233,10 +233,34 @@ alongside CKAN. To install this please look at the docs here: http://docs.ckan.org/projects/datapusher +.. _data_dictionary: + +--------------- +Data Dictionary +--------------- + +DataStore columns may be described with a Data Dictionary. A Data Dictionary tab +will appear when editing any resource with a DataStore table. +The Data Dictionary form allows entering the following values for +each column: + +1. Type Override: the type to be used the next time DataPusher is run to load + data into this column +2. Label: a human-friendly label for this column +3. Description: a full description for this column in markdown format + +Extension developers may add new fields to this form by overriding the default +Data Dictionary form template ``datastore/snippets/dictionary_form.html``. + +The Data Dictionary is set through the API as part of the :ref:`fields` passed +to :meth:`~ckanext.datastore.logic.action.datastore_create` and +returned from :meth:`~ckanext.datastore.logic.action.datastore_search`. + + .. _dump: --------------------- -Downloading resources +Downloading Resources --------------------- A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. From 43139d636c08d4a63c6c8deabda4a3c77de24b57 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 20:12:26 -0500 Subject: [PATCH 09/13] [#3989] link back from Fields to Data Dictionary --- doc/maintaining/datastore.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index 0fe2db7de2b..9d058b49c60 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -244,10 +244,10 @@ will appear when editing any resource with a DataStore table. The Data Dictionary form allows entering the following values for each column: -1. Type Override: the type to be used the next time DataPusher is run to load - data into this column -2. Label: a human-friendly label for this column -3. Description: a full description for this column in markdown format +* **Type Override:** the type to be used the next time DataPusher is run to load + data into this column +* **Label:** a human-friendly label for this column +* **Description:** a full description for this column in markdown format Extension developers may add new fields to this form by overriding the default Data Dictionary form template ``datastore/snippets/dictionary_form.html``. @@ -345,7 +345,7 @@ Set the types to ensure that future inserts will not fail because of an incorrec guessed type. See :ref:`valid-types` for details on which types are valid. Extra ``"info"`` field values will be stored along with the column. ``"label"``, -``"notes"`` and ``"type_override"`` can be managed from the default Data Dictionary +``"notes"`` and ``"type_override"`` can be managed from the default :ref:`data_dictionary` form. Additional fields can be stored by customizing the Data Dictionary form or by passing their values to the API directly. From d3c91dae94e5586abe63a8db5bcd6fff7d0fb4fe Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 10:17:38 +0100 Subject: [PATCH 10/13] Upgrade Solr Docker image Bump to patch release to include security fixes --- contrib/docker/solr/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index baa0acc1899..801a3963ac9 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -1,4 +1,4 @@ -FROM solr:6.2 +FROM solr:6.2.2 MAINTAINER Open Knowledge # Enviroment From fb982dea8c4279968d6ba586e7eae25720a9c00c Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 10:27:49 +0100 Subject: [PATCH 11/13] Fix wrong version --- contrib/docker/solr/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index 801a3963ac9..16c6aee0feb 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -1,4 +1,4 @@ -FROM solr:6.2.2 +FROM solr:6.6.2 MAINTAINER Open Knowledge # Enviroment From 272b59e4eb953bc5656ebbba54d66307a74a49c9 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 10:35:51 +0100 Subject: [PATCH 12/13] Upgrade scripts version --- contrib/docker/solr/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index 16c6aee0feb..9dc8c2063f6 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -14,11 +14,11 @@ RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data # Adding Files ADD ./contrib/docker/solr/solrconfig.xml \ ./ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ /opt/solr/server/solr/$SOLR_CORE/conf/ # Create Core.properties From 49ba9f3ad455933a4bed0d29377ae7db0a1a506a Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 11:22:20 +0100 Subject: [PATCH 13/13] Run as Solr user, as this version won't start as root by default --- contrib/docker/solr/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index 9dc8c2063f6..59ce3492f42 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -4,9 +4,6 @@ MAINTAINER Open Knowledge # Enviroment ENV SOLR_CORE ckan -# User -USER root - # Create Directories RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data @@ -25,4 +22,9 @@ https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/ RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties # Giving ownership to Solr + +USER root RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE + +# User +USER $SOLR_USER:$SOLR_USER