diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py
index c70e4b612c9..968dce61bf4 100644
--- a/ckan/controllers/group.py
+++ b/ckan/controllers/group.py
@@ -5,6 +5,7 @@
from urllib import urlencode
from pylons.i18n import get_lang
+from six import string_types
import ckan.lib.base as base
import ckan.lib.helpers as h
@@ -268,7 +269,7 @@ def search_url(params):
controller = lookup_group_controller(group_type)
action = 'bulk_process' if c.action == 'bulk_process' else 'read'
url = h.url_for(controller=controller, action=action, id=id)
- params = [(k, v.encode('utf-8') if isinstance(v, basestring)
+ params = [(k, v.encode('utf-8') if isinstance(v, string_types)
else str(v)) for k, v in params]
return url + u'?' + urlencode(params)
diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index 2c4a2e85914..fbb313a82f9 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -9,6 +9,7 @@
from ckan.common import config
from paste.deploy.converters import asbool
import paste.fileapp
+from six import string_types
import ckan.logic as logic
import ckan.lib.base as base
@@ -45,7 +46,7 @@
def _encode_params(params):
- return [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
+ return [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v))
for k, v in params]
diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py
index 60476666682..790bfe23c95 100644
--- a/ckan/lib/dictization/model_save.py
+++ b/ckan/lib/dictization/model_save.py
@@ -5,6 +5,7 @@
import logging
from sqlalchemy.orm import class_mapper
+from six import string_types
import ckan.lib.dictization as d
import ckan.lib.helpers as h
@@ -460,7 +461,7 @@ def package_api_to_dict(api1_dict, context):
for key, value in api1_dict.iteritems():
new_value = value
if key == 'tags':
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
new_value = [{"name": item} for item in value.split()]
else:
new_value = [{"name": item} for item in value]
diff --git a/ckan/lib/extract.py b/ckan/lib/extract.py
index 927e53f5ca2..6a57ce664ac 100644
--- a/ckan/lib/extract.py
+++ b/ckan/lib/extract.py
@@ -3,6 +3,7 @@
import re
from jinja2.ext import babel_extract as extract_jinja2
import lib.jinja_extensions
+from six import string_types
jinja_extensions = '''
jinja2.ext.do, jinja2.ext.with_,
@@ -24,7 +25,7 @@ def jinja2_cleaner(fileobj, *args, **kw):
for lineno, func, message, finder in raw_extract:
- if isinstance(message, basestring):
+ if isinstance(message, string_types):
message = lib.jinja_extensions.regularise_html(message)
elif message is not None:
message = (lib.jinja_extensions.regularise_html(message[0])
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index 46f98164da7..523cc95547d 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -35,6 +35,7 @@
from flask import url_for as _flask_default_url_for
from werkzeug.routing import BuildError as FlaskRouteBuildError
import i18n
+from six import string_types
import ckan.exceptions
import ckan.model as model
@@ -111,7 +112,7 @@ def _datestamp_to_datetime(datetime_):
:rtype: datetime
'''
- if isinstance(datetime_, basestring):
+ if isinstance(datetime_, string_types):
try:
datetime_ = date_str_to_datetime(datetime_)
except TypeError:
@@ -171,7 +172,7 @@ def redirect_to(*args, **kw):
_url = ''
skip_url_parsing = False
parse_url = kw.pop('parse_url', False)
- if uargs and len(uargs) is 1 and isinstance(uargs[0], basestring) \
+ if uargs and len(uargs) is 1 and isinstance(uargs[0], string_types) \
and (uargs[0].startswith('/') or is_url(uargs[0])) \
and parse_url is False:
skip_url_parsing = True
@@ -358,7 +359,7 @@ def _url_for_flask(*args, **kw):
# The API routes used to require a slash on the version number, make sure
# we remove it
if (args and args[0].startswith('api.') and
- isinstance(kw.get('ver'), basestring) and
+ isinstance(kw.get('ver'), string_types) and
kw['ver'].startswith('/')):
kw['ver'] = kw['ver'].replace('/', '')
@@ -1051,7 +1052,7 @@ def get_param_int(name, default=10):
def _url_with_params(url, params):
if not params:
return url
- params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v))
+ params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v))
for k, v in params]
return url + u'?' + urlencode(params)
@@ -1802,7 +1803,7 @@ def remove_url_param(key, value=None, replace=None, controller=None,
instead.
'''
- if isinstance(key, basestring):
+ if isinstance(key, string_types):
keys = [key]
else:
keys = key
@@ -2137,7 +2138,7 @@ def format_resource_items(items):
# Sometimes values that can't be converted to ints can sneak
# into the db. In this case, just leave them as they are.
pass
- elif isinstance(value, basestring):
+ elif isinstance(value, string_types):
# check if strings are actually datetime/number etc
if re.search(reg_ex_datetime, value):
datetime_ = date_str_to_datetime(value)
@@ -2534,7 +2535,7 @@ def get_translated(data_dict, field):
return data_dict[field + u'_translated'][language]
except KeyError:
val = data_dict.get(field, '')
- return _(val) if val and isinstance(val, basestring) else val
+ return _(val) if val and isinstance(val, string_types) else val
@core_helper
diff --git a/ckan/lib/search/common.py b/ckan/lib/search/common.py
index 8d031891760..060f62f4175 100644
--- a/ckan/lib/search/common.py
+++ b/ckan/lib/search/common.py
@@ -5,6 +5,8 @@
import re
import pysolr
import simplejson
+from six import string_types
+
log = logging.getLogger(__name__)
@@ -72,7 +74,7 @@ def make_connection(decode_dates=True):
def solr_datetime_decoder(d):
for k, v in d.items():
- if isinstance(v, basestring):
+ if isinstance(v, string_types):
possible_datetime = re.search(pysolr.DATETIME_REGEX, v)
if possible_datetime:
date_values = possible_datetime.groupdict()
diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py
index 13d75e7a799..9ad3adf0b1c 100644
--- a/ckan/lib/search/query.py
+++ b/ckan/lib/search/query.py
@@ -49,7 +49,7 @@ def convert_legacy_parameters_to_solr(legacy_params):
non_solr_params = set(legacy_params.keys()) - VALID_SOLR_PARAMETERS
for search_key in non_solr_params:
value_obj = legacy_params[search_key]
- value = value_obj.replace('+', ' ') if isinstance(value_obj, basestring) else value_obj
+ value = value_obj.replace('+', ' ') if isinstance(value_obj, six.string_types) else value_obj
if search_key == 'all_fields':
if value:
solr_params['fl'] = '*'
@@ -62,7 +62,7 @@ def convert_legacy_parameters_to_solr(legacy_params):
elif search_key == 'tags':
if isinstance(value_obj, list):
tag_list = value_obj
- elif isinstance(value_obj, basestring):
+ elif isinstance(value_obj, six.string_types):
tag_list = [value_obj]
else:
raise SearchQueryError('Was expecting either a string or JSON list for the tags parameter: %r' % value)
@@ -173,7 +173,7 @@ def run(self, query=None, fields=None, options=None, **kwargs):
else:
options.update(kwargs)
- if isinstance(query, basestring):
+ if isinstance(query, six.string_types):
query = [query]
query = query[:] # don't alter caller's query list.
@@ -220,7 +220,7 @@ def run(self, fields={}, options=None, **kwargs):
# action.
query = []
for field, terms in fields.items():
- if isinstance(terms, basestring):
+ if isinstance(terms, six.string_types):
terms = terms.split()
for term in terms:
query.append(':'.join([field, term]))
diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py
index e5dee92206e..fdbf527806e 100644
--- a/ckan/logic/__init__.py
+++ b/ckan/logic/__init__.py
@@ -7,6 +7,7 @@
from collections import defaultdict
import formencode.validators
+from six import string_types
import ckan.model as model
import ckan.authz as authz
@@ -175,7 +176,7 @@ def clean_dict(data_dict):
if not isinstance(value, list):
continue
for inner_dict in value[:]:
- if isinstance(inner_dict, basestring):
+ if isinstance(inner_dict, string_types):
break
if not any(inner_dict.values()):
value.remove(inner_dict)
@@ -516,7 +517,7 @@ def get_or_bust(data_dict, keys):
not in the given dictionary
'''
- if isinstance(keys, basestring):
+ if isinstance(keys, string_types):
keys = [keys]
import ckan.logic.schema as schema
diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py
index d68d3b310d2..a2c9e0fcbf9 100644
--- a/ckan/logic/action/get.py
+++ b/ckan/logic/action/get.py
@@ -11,6 +11,7 @@
from ckan.common import config
import sqlalchemy
from paste.deploy.converters import asbool
+from six import string_types
import ckan.lib.dictization
import ckan.logic as logic
@@ -2109,7 +2110,7 @@ def resource_search(context, data_dict):
{'fields': _('Do not specify if using "query" parameter')})
elif query is not None:
- if isinstance(query, basestring):
+ if isinstance(query, string_types):
query = [query]
try:
fields = dict(pair.split(":", 1) for pair in query)
@@ -2125,7 +2126,7 @@ def resource_search(context, data_dict):
# So maintain that behaviour
split_terms = {}
for field, terms in fields.items():
- if isinstance(terms, basestring):
+ if isinstance(terms, string_types):
terms = terms.split()
split_terms[field] = terms
fields = split_terms
@@ -2143,7 +2144,7 @@ def resource_search(context, data_dict):
resource_fields = model.Resource.get_columns()
for field, terms in fields.items():
- if isinstance(terms, basestring):
+ if isinstance(terms, string_types):
terms = [terms]
if field not in resource_fields:
@@ -2215,7 +2216,7 @@ def _tag_search(context, data_dict):
model = context['model']
terms = data_dict.get('query') or data_dict.get('q') or []
- if isinstance(terms, basestring):
+ if isinstance(terms, string_types):
terms = [terms]
terms = [t.strip() for t in terms if t.strip()]
@@ -2402,7 +2403,7 @@ def term_translation_show(context, data_dict):
# This action accepts `terms` as either a list of strings, or a single
# string.
terms = _get_or_bust(data_dict, 'terms')
- if isinstance(terms, basestring):
+ if isinstance(terms, string_types):
terms = [terms]
if terms:
q = q.where(trans_table.c.term.in_(terms))
@@ -2411,7 +2412,7 @@ def term_translation_show(context, data_dict):
# string.
if 'lang_codes' in data_dict:
lang_codes = _get_or_bust(data_dict, 'lang_codes')
- if isinstance(lang_codes, basestring):
+ if isinstance(lang_codes, string_types):
lang_codes = [lang_codes]
q = q.where(trans_table.c.lang_code.in_(lang_codes))
diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py
index c747555e910..ed97f67de73 100644
--- a/ckan/logic/converters.py
+++ b/ckan/logic/converters.py
@@ -2,6 +2,8 @@
import json
+from six import string_types
+
import ckan.model as model
import ckan.lib.navl.dictization_functions as df
import ckan.logic.validators as validators
@@ -60,7 +62,7 @@ def callable(key, data, errors, context):
new_tags = data.get(key)
if not new_tags:
return
- if isinstance(new_tags, basestring):
+ if isinstance(new_tags, string_types):
new_tags = [new_tags]
# get current number of tags
@@ -173,7 +175,7 @@ def convert_group_name_or_id_to_id(group_name_or_id, context):
def convert_to_json_if_string(value, context):
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
try:
return json.loads(value)
except ValueError:
@@ -183,13 +185,13 @@ def convert_to_json_if_string(value, context):
def convert_to_list_if_string(value, context=None):
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
return [value]
else:
return value
def remove_whitespace(value, context):
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
return value.strip()
return value
diff --git a/ckan/model/types.py b/ckan/model/types.py
index b05fcf335eb..c4351730126 100644
--- a/ckan/model/types.py
+++ b/ckan/model/types.py
@@ -7,6 +7,7 @@
import simplejson as json
from sqlalchemy import types
+from six import string_types
import meta
@@ -74,7 +75,7 @@ def process_bind_param(self, value, engine):
if value is None or value == {}: # ensure we stores nulls in db not json "null"
return None
else:
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
return unicode(value)
else:
return unicode(json.dumps(value, ensure_ascii=False))
@@ -89,7 +90,7 @@ def iso_date_to_datetime_for_sqlite(datetime_or_iso_date_if_sqlite):
# to call this to convert it into a datetime type. When running on
# postgres then you have a datetime anyway, so this function doesn't
# do anything.
- if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, basestring):
+ if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, string_types):
return datetime.datetime.strptime(datetime_or_iso_date_if_sqlite,
'%Y-%m-%d %H:%M:%S.%f')
else:
diff --git a/ckan/plugins/core.py b/ckan/plugins/core.py
index 07456fc8803..14098016d0d 100644
--- a/ckan/plugins/core.py
+++ b/ckan/plugins/core.py
@@ -12,6 +12,7 @@
from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin
from pyutilib.component.core import Plugin as _pca_Plugin
from paste.deploy.converters import asbool
+from six import string_types
import interfaces
@@ -244,7 +245,7 @@ def _get_service(plugin_name):
:return: the service object
'''
- if isinstance(plugin_name, basestring):
+ if isinstance(plugin_name, string_types):
for group in GROUPS:
iterator = iter_entry_points(
group=group,
diff --git a/ckan/tests/legacy/html_check.py b/ckan/tests/legacy/html_check.py
index 5b549942ee6..e8bc05e502b 100644
--- a/ckan/tests/legacy/html_check.py
+++ b/ckan/tests/legacy/html_check.py
@@ -3,13 +3,15 @@
import re
import sgmllib
+from six import string_types
+
import paste.fixture
class HtmlCheckMethods(object):
'''A collection of methods to check properties of a html page, usually
in the form returned by paster.'''
-
+
def named_div(self, div_name, html):
'strips html to just the
section'
the_html = self._get_html_from_res(html)
@@ -31,15 +33,15 @@ def sidebar(self, html):
def strip_tags(self, res):
'''Call strip_tags on a TestResponse object to strip any and all HTML and normalise whitespace.'''
- if not isinstance(res, basestring):
+ if not isinstance(res, string_types):
res = res.body.decode('utf-8')
- return Stripper().strip(res)
+ return Stripper().strip(res)
def check_named_element(self, html, tag_name, *html_to_find):
'''Searches in the html and returns True if it can find a particular
tag and all its subtags & data which contains all the of the
html_to_find'''
- named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?%(tag)s)?>)' % {'tag':tag_name})
+ named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?%(tag)s)?>)' % {'tag':tag_name})
html_str = self._get_html_from_res(html)
self._check_html(named_element_re, html_str.replace('\n', ''), html_to_find)
diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
index c0c287a9c21..7c4118dc5b8 100644
--- a/ckanext/datastore/backend/postgres.py
+++ b/ckanext/datastore/backend/postgres.py
@@ -15,6 +15,8 @@
import json
from cStringIO import StringIO
+from six import string_types
+
import ckan.lib.cli as cli
import ckan.plugins as p
import ckan.plugins.toolkit as toolkit
@@ -374,7 +376,7 @@ def _where_clauses(data_dict, fields_types):
# add full-text search where clause
q = data_dict.get('q')
if q:
- if isinstance(q, basestring):
+ if isinstance(q, string_types):
ts_query_alias = _ts_query_alias()
clause_str = u'_full_text @@ {0}'.format(ts_query_alias)
clauses.append((clause_str,))
@@ -409,7 +411,7 @@ def _textsearch_query(data_dict):
statements = []
rank_columns = []
plain = data_dict.get('plain', True)
- if isinstance(q, basestring):
+ if isinstance(q, string_types):
query, rank = _build_query_and_rank_statements(
lang, q, plain)
statements.append(query)
@@ -469,7 +471,7 @@ def _sort(data_dict, fields_types):
if not sort:
q = data_dict.get('q')
if q:
- if isinstance(q, basestring):
+ if isinstance(q, string_types):
return [_ts_rank_alias()]
elif isinstance(q, dict):
return [_ts_rank_alias(field) for field in q
@@ -1196,7 +1198,7 @@ def validate(context, data_dict):
for key, values in data_dict_copy.iteritems():
if not values:
continue
- if isinstance(values, basestring):
+ if isinstance(values, string_types):
value = values
elif isinstance(values, (list, tuple)):
value = values[0]
diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py
index b616f0f9405..523bb096774 100644
--- a/ckanext/datastore/helpers.py
+++ b/ckanext/datastore/helpers.py
@@ -6,6 +6,8 @@
import paste.deploy.converters as converters
import sqlparse
+from six import string_types
+
from ckan.plugins.toolkit import get_action, ObjectNotFound, NotAuthorized
log = logging.getLogger(__name__)
@@ -57,10 +59,10 @@ def validate_int(i, non_negative=False):
return i >= 0 or not non_negative
-def _strip(input):
- if isinstance(input, basestring) and len(input) and input[0] == input[-1]:
- return input.strip().strip('"')
- return input
+def _strip(s):
+ if isinstance(s, string_types) and len(s) and s[0] == s[-1]:
+ return s.strip().strip('"')
+ return s
def should_fts_index_field_type(field_type):
diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py
index 903e016d71c..9a7d5795925 100644
--- a/ckanext/datastore/logic/schema.py
+++ b/ckanext/datastore/logic/schema.py
@@ -2,6 +2,8 @@
import json
+from six import string_types
+
import ckan.plugins as p
import ckan.lib.navl.dictization_functions as df
@@ -48,13 +50,13 @@ def list_of_strings_or_lists(key, data, errors, context):
if not isinstance(value, list):
raise df.Invalid('Not a list')
for x in value:
- if not isinstance(x, basestring) and not isinstance(x, list):
+ if not isinstance(x, string_types) and not isinstance(x, list):
raise df.Invalid('%s: %s' % ('Neither a string nor a list', x))
def list_of_strings_or_string(key, data, errors, context):
value = data.get(key)
- if isinstance(value, basestring):
+ if isinstance(value, string_types):
return
list_of_strings_or_lists(key, data, errors, context)
diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py
index f47cbbfd796..8d0c6b7b584 100644
--- a/ckanext/datastore/plugin.py
+++ b/ckanext/datastore/plugin.py
@@ -2,6 +2,7 @@
import logging
+from six import string_types
import ckan.plugins as p
import ckan.logic as logic
@@ -213,16 +214,17 @@ def datastore_validate(self, context, data_dict, fields_types):
q = data_dict.get('q')
if q:
- if isinstance(q, basestring):
+ if isinstance(q, string_types):
del data_dict['q']
elif isinstance(q, dict):
for key in q.keys():
- if key in fields_types and isinstance(q[key], basestring):
+ if key in fields_types and isinstance(q[key],
+ string_types):
del q[key]
language = data_dict.get('language')
if language:
- if isinstance(language, basestring):
+ if isinstance(language, string_types):
del data_dict['language']
plain = data_dict.get('plain')
@@ -249,7 +251,7 @@ def datastore_validate(self, context, data_dict, fields_types):
if limit:
is_positive_int = datastore_helpers.validate_int(limit,
non_negative=True)
- is_all = isinstance(limit, basestring) and limit.lower() == 'all'
+ is_all = isinstance(limit, string_types) and limit.lower() == 'all'
if is_positive_int or is_all:
del data_dict['limit']
diff --git a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html
index fad1d662df7..41266c9aae9 100644
--- a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html
+++ b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html
@@ -1,16 +1,25 @@
{% import 'macros/form.html' as form %}
{{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})
- {{ form.select('info__' ~ position ~ '__type_override',
- label=_('Type Override'), options=[
- {'name': '', 'value': ''},
- {'name': 'text', 'value': 'text'},
- {'name': 'numeric', 'value': 'numeric'},
- {'name': 'timestamp', 'value': 'timestamp'},
- ], selected=field.get('info', {}).get('type_override', '')) }}
- {{ form.input('info__' ~ position ~ '__label',
- label=_('Label'), id='field-f' ~ position ~ 'label',
- value=field.get('info', {}).get('label', ''), classes=['control-full']) }}
- {{ form.markdown('info__' ~ position ~ '__notes',
- label=_('Description'), id='field-d' ~ position ~ 'notes',
- value=field.get('info', {}).get('notes', '')) }}
+
+{#
+ Data Dictionary fields may be added this snippet. New fields following
+ the 'info__' ~ position ~ '__namegoeshere' convention will be saved
+ as part of the "info" object on the column.
+#}
+
+{{ form.select('info__' ~ position ~ '__type_override',
+ label=_('Type Override'), options=[
+ {'name': '', 'value': ''},
+ {'name': 'text', 'value': 'text'},
+ {'name': 'numeric', 'value': 'numeric'},
+ {'name': 'timestamp', 'value': 'timestamp'},
+ ], selected=field.get('info', {}).get('type_override', '')) }}
+
+{{ form.input('info__' ~ position ~ '__label',
+ label=_('Label'), id='field-f' ~ position ~ 'label',
+ value=field.get('info', {}).get('label', ''), classes=['control-full']) }}
+
+{{ form.markdown('info__' ~ position ~ '__notes',
+ label=_('Description'), id='field-d' ~ position ~ 'notes',
+ value=field.get('info', {}).get('notes', '')) }}
diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py
index 238aa5e7371..c4615151572 100644
--- a/ckanext/multilingual/plugin.py
+++ b/ckanext/multilingual/plugin.py
@@ -1,5 +1,7 @@
# encoding: utf-8
+from six import string_types
+
import ckan
from ckan.plugins import SingletonPlugin, implements, IPackageController
from ckan.plugins import IGroupController, IOrganizationController, ITagController, IResourceController
@@ -31,7 +33,7 @@ def translate_data_dict(data_dict):
for (key, value) in flattened.items():
if value in (None, True, False):
continue
- elif isinstance(value, basestring):
+ elif isinstance(value, string_types):
terms.add(value)
elif isinstance(value, (int, long)):
continue
@@ -79,7 +81,7 @@ def translate_data_dict(data_dict):
# Don't try to translate values that aren't strings.
translated_flattened[key] = value
- elif isinstance(value, basestring):
+ elif isinstance(value, string_types):
if value in desired_translations:
translated_flattened[key] = desired_translations[value]
else:
@@ -127,7 +129,7 @@ def translate_resource_data_dict(data_dict):
for (key, value) in flattened.items():
if value in (None, True, False):
continue
- elif isinstance(value, basestring):
+ elif isinstance(value, string_types):
terms.add(value)
elif isinstance(value, (int, long)):
continue
@@ -170,7 +172,7 @@ def translate_resource_data_dict(data_dict):
# Don't try to translate values that aren't strings.
translated_flattened[key] = value
- elif isinstance(value, basestring):
+ elif isinstance(value, string_types):
if value in desired_translations:
translated_flattened[key] = desired_translations[value]
else:
@@ -229,7 +231,7 @@ def before_index(self, search_data):
if not isinstance(value, list):
value = [value]
for item in value:
- if isinstance(item, basestring):
+ if isinstance(item, string_types):
all_terms.append(item)
field_translations = get_action('term_translation_show')(
diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile
index baa0acc1899..59ce3492f42 100644
--- a/contrib/docker/solr/Dockerfile
+++ b/contrib/docker/solr/Dockerfile
@@ -1,12 +1,9 @@
-FROM solr:6.2
+FROM solr:6.6.2
MAINTAINER Open Knowledge
# Enviroment
ENV SOLR_CORE ckan
-# User
-USER root
-
# Create Directories
RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf
RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data
@@ -14,15 +11,20 @@ RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data
# Adding Files
ADD ./contrib/docker/solr/solrconfig.xml \
./ckan/config/solr/schema.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
-https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/currency.xml \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/protwords.txt \
+https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \
/opt/solr/server/solr/$SOLR_CORE/conf/
# Create Core.properties
RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties
# Giving ownership to Solr
+
+USER root
RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE
+
+# User
+USER $SOLR_USER:$SOLR_USER
diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst
index c4da16b5d16..9d058b49c60 100644
--- a/doc/maintaining/datastore.rst
+++ b/doc/maintaining/datastore.rst
@@ -233,6 +233,49 @@ alongside CKAN.
To install this please look at the docs here: http://docs.ckan.org/projects/datapusher
+.. _data_dictionary:
+
+---------------
+Data Dictionary
+---------------
+
+DataStore columns may be described with a Data Dictionary. A Data Dictionary tab
+will appear when editing any resource with a DataStore table.
+The Data Dictionary form allows entering the following values for
+each column:
+
+* **Type Override:** the type to be used the next time DataPusher is run to load
+ data into this column
+* **Label:** a human-friendly label for this column
+* **Description:** a full description for this column in markdown format
+
+Extension developers may add new fields to this form by overriding the default
+Data Dictionary form template ``datastore/snippets/dictionary_form.html``.
+
+The Data Dictionary is set through the API as part of the :ref:`fields` passed
+to :meth:`~ckanext.datastore.logic.action.datastore_create` and
+returned from :meth:`~ckanext.datastore.logic.action.datastore_search`.
+
+
+.. _dump:
+
+---------------------
+Downloading Resources
+---------------------
+
+A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``.
+
+For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``.
+
+Other formats supported include tab-separated values (``?format=tsv``),
+JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible
+tab-separated file use
+``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``.
+
+.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values
+
+
+
-----------------
The DataStore API
-----------------
@@ -279,23 +322,6 @@ API reference
:members:
-.. _dump:
-
-Download resource
------------------
-
-A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``.
-
-For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``.
-
-Other formats supported include tab-separated values (``?format=tsv``),
-JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible
-tab-separated file use
-``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``.
-
-.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values
-
-
.. _fields:
Fields
@@ -304,22 +330,40 @@ Fields
Fields define the column names and the type of the data in a column. A field is defined as follows::
{
- "id": # a string which defines the column name
+ "id": # the column name (required)
"type": # the data type for the column
+ "info": {
+ "label": # human-readable label for column
+ "notes": # markdown description of column
+ "type_override": # type for datapusher to use when importing data
+ ...: # other user-defined fields
+ }
}
-Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid.
+Field types not provided will be guessed based on the first row of provided data.
+Set the types to ensure that future inserts will not fail because of an incorrectly
+guessed type. See :ref:`valid-types` for details on which types are valid.
+
+Extra ``"info"`` field values will be stored along with the column. ``"label"``,
+``"notes"`` and ``"type_override"`` can be managed from the default :ref:`data_dictionary`
+form. Additional fields can be stored by customizing the Data Dictionary form or by
+passing their values to the API directly.
Example::
[
{
- "id": "foo",
- "type": "int4"
+ "id": "code_number",
+ "type": "numeric"
},
{
- "id": "bar"
- # type is optional
+ "id": "description"
+ "type": "text",
+ "info": {
+ "label": "Description",
+ "notes": "A brief usage description for this code",
+ "example": "Used for temporary service interruptions"
+ }
}
]
@@ -331,19 +375,21 @@ Records
A record is the data to be inserted in a DataStore resource and is defined as follows::
{
- "": # data to be set
- # .. more data
+ column_1_id: value_1,
+ columd_2_id: value_2,
+ ...
}
Example::
[
{
- "foo": 100,
- "bar": "Here's some text"
+ "code_number": 10,
+ "description": "Submitted successfully"
},
{
- "foo": 42
+ "code_number": 42,
+ "description": "In progress"
}
]
@@ -437,8 +483,10 @@ name
oid
The PostgreSQL object ID of the table that belongs to name.
+
+-------------------
Extending DataStore
-===================
+-------------------
Starting from CKAN version 2.7, backend used in DataStore can be replaced with custom one. For this purpose, custom extension must implement `ckanext.datastore.interfaces.IDatastoreBackend`, which provides one method - `register_backends`. It should return dictonary with names of custom backends as keys and classes, that represent those backends as values. Each class supposed to be inherited from `ckanext.datastore.backend.DatastoreBackend`.
diff --git a/requirements.in b/requirements.in
index 97a2353ccbe..6cc687dfc8d 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,11 +1,10 @@
# The file contains the direct ckan requirements.
# Use pip-compile to create a requirements.txt file from this
Babel==2.3.4
-Beaker==1.9.0 # Needs to be pinned to a more up to date version than the Pylons one
bleach==2.1.2
click==6.7
fanstatic==0.12
-Flask==0.11.1
+Flask==0.12.2
Flask-Babel==0.11.2
Jinja2==2.8
Markdown==2.6.7
diff --git a/requirements.txt b/requirements.txt
index 9eedbfe53a9..7c518a6906a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -5,22 +5,22 @@
# pip-compile --output-file requirements.txt requirements.in
#
argparse==1.4.0 # via ofs
-Babel==2.3.4
+Babel==2.3.4 # via flask-babel
Beaker==1.9.0 # via pylons
bleach==2.1.2
click==6.7
-decorator==4.0.6 # via pylons, sqlalchemy-migrate
+decorator==4.2.1 # via pylons, sqlalchemy-migrate
fanstatic==0.12
-Flask==0.11.1
Flask-Babel==0.11.2
-FormEncode==1.3.0 # via pylons
+Flask==0.12.2
+FormEncode==1.3.1 # via pylons
funcsigs==1.0.2 # via beaker
html5lib==1.0.1 # via bleach
itsdangerous==0.24 # via flask
-Jinja2==2.8 # via flask
-Mako==1.0.4 # via pylons
+Jinja2==2.8 # via flask, flask-babel
+Mako==1.0.7 # via pylons
Markdown==2.6.7
-MarkupSafe==0.23 # via jinja2, mako, webhelpers
+MarkupSafe==1.0 # via jinja2, mako, webhelpers
nose==1.3.7 # via pylons
ofs==0.4.2
Pairtree==0.7.1-T
@@ -31,22 +31,22 @@ PasteScript==2.0.2 # via pylons
pbr==1.10.0 # via sqlalchemy-migrate
polib==1.0.7
psycopg2==2.7.3.2
-Pygments==2.1.3 # via weberror
+Pygments==2.2.0 # via weberror
Pylons==0.9.7
pysolr==3.6.0
python-dateutil==1.5
python-magic==0.4.12
pytz==2016.7
pyutilib.component.core==4.6.4
-redis==2.10.5 # via rq
-repoze.lru==0.6 # via routes
+redis==2.10.6 # via rq
+repoze.lru==0.7 # via routes
repoze.who-friendlyform==1.0.8
repoze.who==2.3
requests==2.11.1
Routes==1.13 # via pylons
rq==0.6.0
simplejson==3.10.0
-six==1.10.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate
+six==1.11.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate
sqlalchemy-migrate==0.10.0
SQLAlchemy==1.1.11 # via sqlalchemy-migrate
sqlparse==0.2.2
@@ -59,7 +59,7 @@ WebError==0.13.1 # via pylons
WebHelpers==1.3 # via pylons
WebOb==1.0.8 # via fanstatic, pylons, repoze.who, repoze.who-friendlyform, weberror, webtest
WebTest==1.4.3 # via pylons
-Werkzeug==0.11.10 # via flask
+Werkzeug==0.14.1 # via flask
zope.interface==4.3.2
# The following packages are commented out because they are