diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index c70e4b612c9..968dce61bf4 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -5,6 +5,7 @@ from urllib import urlencode from pylons.i18n import get_lang +from six import string_types import ckan.lib.base as base import ckan.lib.helpers as h @@ -268,7 +269,7 @@ def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) - params = [(k, v.encode('utf-8') if isinstance(v, basestring) + params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 2c4a2e85914..fbb313a82f9 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -9,6 +9,7 @@ from ckan.common import config from paste.deploy.converters import asbool import paste.fileapp +from six import string_types import ckan.logic as logic import ckan.lib.base as base @@ -45,7 +46,7 @@ def _encode_params(params): - return [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) + return [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index 60476666682..790bfe23c95 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -5,6 +5,7 @@ import logging from sqlalchemy.orm import class_mapper +from six import string_types import ckan.lib.dictization as d import ckan.lib.helpers as h @@ -460,7 +461,7 @@ def package_api_to_dict(api1_dict, context): for key, value in api1_dict.iteritems(): new_value = value if key == 'tags': - if isinstance(value, basestring): + if isinstance(value, string_types): new_value = [{"name": item} for item in value.split()] else: new_value = [{"name": item} for item in value] diff --git a/ckan/lib/extract.py b/ckan/lib/extract.py index 927e53f5ca2..6a57ce664ac 100644 --- a/ckan/lib/extract.py +++ b/ckan/lib/extract.py @@ -3,6 +3,7 @@ import re from jinja2.ext import babel_extract as extract_jinja2 import lib.jinja_extensions +from six import string_types jinja_extensions = ''' jinja2.ext.do, jinja2.ext.with_, @@ -24,7 +25,7 @@ def jinja2_cleaner(fileobj, *args, **kw): for lineno, func, message, finder in raw_extract: - if isinstance(message, basestring): + if isinstance(message, string_types): message = lib.jinja_extensions.regularise_html(message) elif message is not None: message = (lib.jinja_extensions.regularise_html(message[0]) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 46f98164da7..523cc95547d 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -35,6 +35,7 @@ from flask import url_for as _flask_default_url_for from werkzeug.routing import BuildError as FlaskRouteBuildError import i18n +from six import string_types import ckan.exceptions import ckan.model as model @@ -111,7 +112,7 @@ def _datestamp_to_datetime(datetime_): :rtype: datetime ''' - if isinstance(datetime_, basestring): + if isinstance(datetime_, string_types): try: datetime_ = date_str_to_datetime(datetime_) except TypeError: @@ -171,7 +172,7 @@ def redirect_to(*args, **kw): _url = '' skip_url_parsing = False parse_url = kw.pop('parse_url', False) - if uargs and len(uargs) is 1 and isinstance(uargs[0], basestring) \ + if uargs and len(uargs) is 1 and isinstance(uargs[0], string_types) \ and (uargs[0].startswith('/') or is_url(uargs[0])) \ and parse_url is False: skip_url_parsing = True @@ -358,7 +359,7 @@ def _url_for_flask(*args, **kw): # The API routes used to require a slash on the version number, make sure # we remove it if (args and args[0].startswith('api.') and - isinstance(kw.get('ver'), basestring) and + isinstance(kw.get('ver'), string_types) and kw['ver'].startswith('/')): kw['ver'] = kw['ver'].replace('/', '') @@ -1051,7 +1052,7 @@ def get_param_int(name, default=10): def _url_with_params(url, params): if not params: return url - params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) + params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) @@ -1802,7 +1803,7 @@ def remove_url_param(key, value=None, replace=None, controller=None, instead. ''' - if isinstance(key, basestring): + if isinstance(key, string_types): keys = [key] else: keys = key @@ -2137,7 +2138,7 @@ def format_resource_items(items): # Sometimes values that can't be converted to ints can sneak # into the db. In this case, just leave them as they are. pass - elif isinstance(value, basestring): + elif isinstance(value, string_types): # check if strings are actually datetime/number etc if re.search(reg_ex_datetime, value): datetime_ = date_str_to_datetime(value) @@ -2534,7 +2535,7 @@ def get_translated(data_dict, field): return data_dict[field + u'_translated'][language] except KeyError: val = data_dict.get(field, '') - return _(val) if val and isinstance(val, basestring) else val + return _(val) if val and isinstance(val, string_types) else val @core_helper diff --git a/ckan/lib/search/common.py b/ckan/lib/search/common.py index 8d031891760..060f62f4175 100644 --- a/ckan/lib/search/common.py +++ b/ckan/lib/search/common.py @@ -5,6 +5,8 @@ import re import pysolr import simplejson +from six import string_types + log = logging.getLogger(__name__) @@ -72,7 +74,7 @@ def make_connection(decode_dates=True): def solr_datetime_decoder(d): for k, v in d.items(): - if isinstance(v, basestring): + if isinstance(v, string_types): possible_datetime = re.search(pysolr.DATETIME_REGEX, v) if possible_datetime: date_values = possible_datetime.groupdict() diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 13d75e7a799..9ad3adf0b1c 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -49,7 +49,7 @@ def convert_legacy_parameters_to_solr(legacy_params): non_solr_params = set(legacy_params.keys()) - VALID_SOLR_PARAMETERS for search_key in non_solr_params: value_obj = legacy_params[search_key] - value = value_obj.replace('+', ' ') if isinstance(value_obj, basestring) else value_obj + value = value_obj.replace('+', ' ') if isinstance(value_obj, six.string_types) else value_obj if search_key == 'all_fields': if value: solr_params['fl'] = '*' @@ -62,7 +62,7 @@ def convert_legacy_parameters_to_solr(legacy_params): elif search_key == 'tags': if isinstance(value_obj, list): tag_list = value_obj - elif isinstance(value_obj, basestring): + elif isinstance(value_obj, six.string_types): tag_list = [value_obj] else: raise SearchQueryError('Was expecting either a string or JSON list for the tags parameter: %r' % value) @@ -173,7 +173,7 @@ def run(self, query=None, fields=None, options=None, **kwargs): else: options.update(kwargs) - if isinstance(query, basestring): + if isinstance(query, six.string_types): query = [query] query = query[:] # don't alter caller's query list. @@ -220,7 +220,7 @@ def run(self, fields={}, options=None, **kwargs): # action. query = [] for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, six.string_types): terms = terms.split() for term in terms: query.append(':'.join([field, term])) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index e5dee92206e..fdbf527806e 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -7,6 +7,7 @@ from collections import defaultdict import formencode.validators +from six import string_types import ckan.model as model import ckan.authz as authz @@ -175,7 +176,7 @@ def clean_dict(data_dict): if not isinstance(value, list): continue for inner_dict in value[:]: - if isinstance(inner_dict, basestring): + if isinstance(inner_dict, string_types): break if not any(inner_dict.values()): value.remove(inner_dict) @@ -516,7 +517,7 @@ def get_or_bust(data_dict, keys): not in the given dictionary ''' - if isinstance(keys, basestring): + if isinstance(keys, string_types): keys = [keys] import ckan.logic.schema as schema diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index d68d3b310d2..a2c9e0fcbf9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -11,6 +11,7 @@ from ckan.common import config import sqlalchemy from paste.deploy.converters import asbool +from six import string_types import ckan.lib.dictization import ckan.logic as logic @@ -2109,7 +2110,7 @@ def resource_search(context, data_dict): {'fields': _('Do not specify if using "query" parameter')}) elif query is not None: - if isinstance(query, basestring): + if isinstance(query, string_types): query = [query] try: fields = dict(pair.split(":", 1) for pair in query) @@ -2125,7 +2126,7 @@ def resource_search(context, data_dict): # So maintain that behaviour split_terms = {} for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = terms.split() split_terms[field] = terms fields = split_terms @@ -2143,7 +2144,7 @@ def resource_search(context, data_dict): resource_fields = model.Resource.get_columns() for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] if field not in resource_fields: @@ -2215,7 +2216,7 @@ def _tag_search(context, data_dict): model = context['model'] terms = data_dict.get('query') or data_dict.get('q') or [] - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] terms = [t.strip() for t in terms if t.strip()] @@ -2402,7 +2403,7 @@ def term_translation_show(context, data_dict): # This action accepts `terms` as either a list of strings, or a single # string. terms = _get_or_bust(data_dict, 'terms') - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] if terms: q = q.where(trans_table.c.term.in_(terms)) @@ -2411,7 +2412,7 @@ def term_translation_show(context, data_dict): # string. if 'lang_codes' in data_dict: lang_codes = _get_or_bust(data_dict, 'lang_codes') - if isinstance(lang_codes, basestring): + if isinstance(lang_codes, string_types): lang_codes = [lang_codes] q = q.where(trans_table.c.lang_code.in_(lang_codes)) diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py index c747555e910..ed97f67de73 100644 --- a/ckan/logic/converters.py +++ b/ckan/logic/converters.py @@ -2,6 +2,8 @@ import json +from six import string_types + import ckan.model as model import ckan.lib.navl.dictization_functions as df import ckan.logic.validators as validators @@ -60,7 +62,7 @@ def callable(key, data, errors, context): new_tags = data.get(key) if not new_tags: return - if isinstance(new_tags, basestring): + if isinstance(new_tags, string_types): new_tags = [new_tags] # get current number of tags @@ -173,7 +175,7 @@ def convert_group_name_or_id_to_id(group_name_or_id, context): def convert_to_json_if_string(value, context): - if isinstance(value, basestring): + if isinstance(value, string_types): try: return json.loads(value) except ValueError: @@ -183,13 +185,13 @@ def convert_to_json_if_string(value, context): def convert_to_list_if_string(value, context=None): - if isinstance(value, basestring): + if isinstance(value, string_types): return [value] else: return value def remove_whitespace(value, context): - if isinstance(value, basestring): + if isinstance(value, string_types): return value.strip() return value diff --git a/ckan/model/types.py b/ckan/model/types.py index b05fcf335eb..c4351730126 100644 --- a/ckan/model/types.py +++ b/ckan/model/types.py @@ -7,6 +7,7 @@ import simplejson as json from sqlalchemy import types +from six import string_types import meta @@ -74,7 +75,7 @@ def process_bind_param(self, value, engine): if value is None or value == {}: # ensure we stores nulls in db not json "null" return None else: - if isinstance(value, basestring): + if isinstance(value, string_types): return unicode(value) else: return unicode(json.dumps(value, ensure_ascii=False)) @@ -89,7 +90,7 @@ def iso_date_to_datetime_for_sqlite(datetime_or_iso_date_if_sqlite): # to call this to convert it into a datetime type. When running on # postgres then you have a datetime anyway, so this function doesn't # do anything. - if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, basestring): + if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, string_types): return datetime.datetime.strptime(datetime_or_iso_date_if_sqlite, '%Y-%m-%d %H:%M:%S.%f') else: diff --git a/ckan/plugins/core.py b/ckan/plugins/core.py index 07456fc8803..14098016d0d 100644 --- a/ckan/plugins/core.py +++ b/ckan/plugins/core.py @@ -12,6 +12,7 @@ from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin from pyutilib.component.core import Plugin as _pca_Plugin from paste.deploy.converters import asbool +from six import string_types import interfaces @@ -244,7 +245,7 @@ def _get_service(plugin_name): :return: the service object ''' - if isinstance(plugin_name, basestring): + if isinstance(plugin_name, string_types): for group in GROUPS: iterator = iter_entry_points( group=group, diff --git a/ckan/tests/legacy/html_check.py b/ckan/tests/legacy/html_check.py index 5b549942ee6..e8bc05e502b 100644 --- a/ckan/tests/legacy/html_check.py +++ b/ckan/tests/legacy/html_check.py @@ -3,13 +3,15 @@ import re import sgmllib +from six import string_types + import paste.fixture class HtmlCheckMethods(object): '''A collection of methods to check properties of a html page, usually in the form returned by paster.''' - + def named_div(self, div_name, html): 'strips html to just the
section' the_html = self._get_html_from_res(html) @@ -31,15 +33,15 @@ def sidebar(self, html): def strip_tags(self, res): '''Call strip_tags on a TestResponse object to strip any and all HTML and normalise whitespace.''' - if not isinstance(res, basestring): + if not isinstance(res, string_types): res = res.body.decode('utf-8') - return Stripper().strip(res) + return Stripper().strip(res) def check_named_element(self, html, tag_name, *html_to_find): '''Searches in the html and returns True if it can find a particular tag and all its subtags & data which contains all the of the html_to_find''' - named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?)' % {'tag':tag_name}) + named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?)' % {'tag':tag_name}) html_str = self._get_html_from_res(html) self._check_html(named_element_re, html_str.replace('\n', ''), html_to_find) diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index c0c287a9c21..7c4118dc5b8 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -15,6 +15,8 @@ import json from cStringIO import StringIO +from six import string_types + import ckan.lib.cli as cli import ckan.plugins as p import ckan.plugins.toolkit as toolkit @@ -374,7 +376,7 @@ def _where_clauses(data_dict, fields_types): # add full-text search where clause q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): ts_query_alias = _ts_query_alias() clause_str = u'_full_text @@ {0}'.format(ts_query_alias) clauses.append((clause_str,)) @@ -409,7 +411,7 @@ def _textsearch_query(data_dict): statements = [] rank_columns = [] plain = data_dict.get('plain', True) - if isinstance(q, basestring): + if isinstance(q, string_types): query, rank = _build_query_and_rank_statements( lang, q, plain) statements.append(query) @@ -469,7 +471,7 @@ def _sort(data_dict, fields_types): if not sort: q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): return [_ts_rank_alias()] elif isinstance(q, dict): return [_ts_rank_alias(field) for field in q @@ -1196,7 +1198,7 @@ def validate(context, data_dict): for key, values in data_dict_copy.iteritems(): if not values: continue - if isinstance(values, basestring): + if isinstance(values, string_types): value = values elif isinstance(values, (list, tuple)): value = values[0] diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py index b616f0f9405..523bb096774 100644 --- a/ckanext/datastore/helpers.py +++ b/ckanext/datastore/helpers.py @@ -6,6 +6,8 @@ import paste.deploy.converters as converters import sqlparse +from six import string_types + from ckan.plugins.toolkit import get_action, ObjectNotFound, NotAuthorized log = logging.getLogger(__name__) @@ -57,10 +59,10 @@ def validate_int(i, non_negative=False): return i >= 0 or not non_negative -def _strip(input): - if isinstance(input, basestring) and len(input) and input[0] == input[-1]: - return input.strip().strip('"') - return input +def _strip(s): + if isinstance(s, string_types) and len(s) and s[0] == s[-1]: + return s.strip().strip('"') + return s def should_fts_index_field_type(field_type): diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 903e016d71c..9a7d5795925 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -2,6 +2,8 @@ import json +from six import string_types + import ckan.plugins as p import ckan.lib.navl.dictization_functions as df @@ -48,13 +50,13 @@ def list_of_strings_or_lists(key, data, errors, context): if not isinstance(value, list): raise df.Invalid('Not a list') for x in value: - if not isinstance(x, basestring) and not isinstance(x, list): + if not isinstance(x, string_types) and not isinstance(x, list): raise df.Invalid('%s: %s' % ('Neither a string nor a list', x)) def list_of_strings_or_string(key, data, errors, context): value = data.get(key) - if isinstance(value, basestring): + if isinstance(value, string_types): return list_of_strings_or_lists(key, data, errors, context) diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index f47cbbfd796..8d0c6b7b584 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -2,6 +2,7 @@ import logging +from six import string_types import ckan.plugins as p import ckan.logic as logic @@ -213,16 +214,17 @@ def datastore_validate(self, context, data_dict, fields_types): q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): del data_dict['q'] elif isinstance(q, dict): for key in q.keys(): - if key in fields_types and isinstance(q[key], basestring): + if key in fields_types and isinstance(q[key], + string_types): del q[key] language = data_dict.get('language') if language: - if isinstance(language, basestring): + if isinstance(language, string_types): del data_dict['language'] plain = data_dict.get('plain') @@ -249,7 +251,7 @@ def datastore_validate(self, context, data_dict, fields_types): if limit: is_positive_int = datastore_helpers.validate_int(limit, non_negative=True) - is_all = isinstance(limit, basestring) and limit.lower() == 'all' + is_all = isinstance(limit, string_types) and limit.lower() == 'all' if is_positive_int or is_all: del data_dict['limit'] diff --git a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html index fad1d662df7..41266c9aae9 100644 --- a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html +++ b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html @@ -1,16 +1,25 @@ {% import 'macros/form.html' as form %}

{{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})

- {{ form.select('info__' ~ position ~ '__type_override', - label=_('Type Override'), options=[ - {'name': '', 'value': ''}, - {'name': 'text', 'value': 'text'}, - {'name': 'numeric', 'value': 'numeric'}, - {'name': 'timestamp', 'value': 'timestamp'}, - ], selected=field.get('info', {}).get('type_override', '')) }} - {{ form.input('info__' ~ position ~ '__label', - label=_('Label'), id='field-f' ~ position ~ 'label', - value=field.get('info', {}).get('label', ''), classes=['control-full']) }} - {{ form.markdown('info__' ~ position ~ '__notes', - label=_('Description'), id='field-d' ~ position ~ 'notes', - value=field.get('info', {}).get('notes', '')) }} + +{# + Data Dictionary fields may be added this snippet. New fields following + the 'info__' ~ position ~ '__namegoeshere' convention will be saved + as part of the "info" object on the column. +#} + +{{ form.select('info__' ~ position ~ '__type_override', + label=_('Type Override'), options=[ + {'name': '', 'value': ''}, + {'name': 'text', 'value': 'text'}, + {'name': 'numeric', 'value': 'numeric'}, + {'name': 'timestamp', 'value': 'timestamp'}, + ], selected=field.get('info', {}).get('type_override', '')) }} + +{{ form.input('info__' ~ position ~ '__label', + label=_('Label'), id='field-f' ~ position ~ 'label', + value=field.get('info', {}).get('label', ''), classes=['control-full']) }} + +{{ form.markdown('info__' ~ position ~ '__notes', + label=_('Description'), id='field-d' ~ position ~ 'notes', + value=field.get('info', {}).get('notes', '')) }} diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py index 238aa5e7371..c4615151572 100644 --- a/ckanext/multilingual/plugin.py +++ b/ckanext/multilingual/plugin.py @@ -1,5 +1,7 @@ # encoding: utf-8 +from six import string_types + import ckan from ckan.plugins import SingletonPlugin, implements, IPackageController from ckan.plugins import IGroupController, IOrganizationController, ITagController, IResourceController @@ -31,7 +33,7 @@ def translate_data_dict(data_dict): for (key, value) in flattened.items(): if value in (None, True, False): continue - elif isinstance(value, basestring): + elif isinstance(value, string_types): terms.add(value) elif isinstance(value, (int, long)): continue @@ -79,7 +81,7 @@ def translate_data_dict(data_dict): # Don't try to translate values that aren't strings. translated_flattened[key] = value - elif isinstance(value, basestring): + elif isinstance(value, string_types): if value in desired_translations: translated_flattened[key] = desired_translations[value] else: @@ -127,7 +129,7 @@ def translate_resource_data_dict(data_dict): for (key, value) in flattened.items(): if value in (None, True, False): continue - elif isinstance(value, basestring): + elif isinstance(value, string_types): terms.add(value) elif isinstance(value, (int, long)): continue @@ -170,7 +172,7 @@ def translate_resource_data_dict(data_dict): # Don't try to translate values that aren't strings. translated_flattened[key] = value - elif isinstance(value, basestring): + elif isinstance(value, string_types): if value in desired_translations: translated_flattened[key] = desired_translations[value] else: @@ -229,7 +231,7 @@ def before_index(self, search_data): if not isinstance(value, list): value = [value] for item in value: - if isinstance(item, basestring): + if isinstance(item, string_types): all_terms.append(item) field_translations = get_action('term_translation_show')( diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index baa0acc1899..59ce3492f42 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -1,12 +1,9 @@ -FROM solr:6.2 +FROM solr:6.6.2 MAINTAINER Open Knowledge # Enviroment ENV SOLR_CORE ckan -# User -USER root - # Create Directories RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data @@ -14,15 +11,20 @@ RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data # Adding Files ADD ./contrib/docker/solr/solrconfig.xml \ ./ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ /opt/solr/server/solr/$SOLR_CORE/conf/ # Create Core.properties RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties # Giving ownership to Solr + +USER root RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE + +# User +USER $SOLR_USER:$SOLR_USER diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index c4da16b5d16..9d058b49c60 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -233,6 +233,49 @@ alongside CKAN. To install this please look at the docs here: http://docs.ckan.org/projects/datapusher +.. _data_dictionary: + +--------------- +Data Dictionary +--------------- + +DataStore columns may be described with a Data Dictionary. A Data Dictionary tab +will appear when editing any resource with a DataStore table. +The Data Dictionary form allows entering the following values for +each column: + +* **Type Override:** the type to be used the next time DataPusher is run to load + data into this column +* **Label:** a human-friendly label for this column +* **Description:** a full description for this column in markdown format + +Extension developers may add new fields to this form by overriding the default +Data Dictionary form template ``datastore/snippets/dictionary_form.html``. + +The Data Dictionary is set through the API as part of the :ref:`fields` passed +to :meth:`~ckanext.datastore.logic.action.datastore_create` and +returned from :meth:`~ckanext.datastore.logic.action.datastore_search`. + + +.. _dump: + +--------------------- +Downloading Resources +--------------------- + +A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. + +For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``. + +Other formats supported include tab-separated values (``?format=tsv``), +JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible +tab-separated file use +``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``. + +.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values + + + ----------------- The DataStore API ----------------- @@ -279,23 +322,6 @@ API reference :members: -.. _dump: - -Download resource ------------------ - -A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. - -For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``. - -Other formats supported include tab-separated values (``?format=tsv``), -JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible -tab-separated file use -``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``. - -.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values - - .. _fields: Fields @@ -304,22 +330,40 @@ Fields Fields define the column names and the type of the data in a column. A field is defined as follows:: { - "id": # a string which defines the column name + "id": # the column name (required) "type": # the data type for the column + "info": { + "label": # human-readable label for column + "notes": # markdown description of column + "type_override": # type for datapusher to use when importing data + ...: # other user-defined fields + } } -Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid. +Field types not provided will be guessed based on the first row of provided data. +Set the types to ensure that future inserts will not fail because of an incorrectly +guessed type. See :ref:`valid-types` for details on which types are valid. + +Extra ``"info"`` field values will be stored along with the column. ``"label"``, +``"notes"`` and ``"type_override"`` can be managed from the default :ref:`data_dictionary` +form. Additional fields can be stored by customizing the Data Dictionary form or by +passing their values to the API directly. Example:: [ { - "id": "foo", - "type": "int4" + "id": "code_number", + "type": "numeric" }, { - "id": "bar" - # type is optional + "id": "description" + "type": "text", + "info": { + "label": "Description", + "notes": "A brief usage description for this code", + "example": "Used for temporary service interruptions" + } } ] @@ -331,19 +375,21 @@ Records A record is the data to be inserted in a DataStore resource and is defined as follows:: { - "": # data to be set - # .. more data + column_1_id: value_1, + columd_2_id: value_2, + ... } Example:: [ { - "foo": 100, - "bar": "Here's some text" + "code_number": 10, + "description": "Submitted successfully" }, { - "foo": 42 + "code_number": 42, + "description": "In progress" } ] @@ -437,8 +483,10 @@ name oid The PostgreSQL object ID of the table that belongs to name. + +------------------- Extending DataStore -=================== +------------------- Starting from CKAN version 2.7, backend used in DataStore can be replaced with custom one. For this purpose, custom extension must implement `ckanext.datastore.interfaces.IDatastoreBackend`, which provides one method - `register_backends`. It should return dictonary with names of custom backends as keys and classes, that represent those backends as values. Each class supposed to be inherited from `ckanext.datastore.backend.DatastoreBackend`. diff --git a/requirements.in b/requirements.in index 97a2353ccbe..6cc687dfc8d 100644 --- a/requirements.in +++ b/requirements.in @@ -1,11 +1,10 @@ # The file contains the direct ckan requirements. # Use pip-compile to create a requirements.txt file from this Babel==2.3.4 -Beaker==1.9.0 # Needs to be pinned to a more up to date version than the Pylons one bleach==2.1.2 click==6.7 fanstatic==0.12 -Flask==0.11.1 +Flask==0.12.2 Flask-Babel==0.11.2 Jinja2==2.8 Markdown==2.6.7 diff --git a/requirements.txt b/requirements.txt index 9eedbfe53a9..7c518a6906a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,22 +5,22 @@ # pip-compile --output-file requirements.txt requirements.in # argparse==1.4.0 # via ofs -Babel==2.3.4 +Babel==2.3.4 # via flask-babel Beaker==1.9.0 # via pylons bleach==2.1.2 click==6.7 -decorator==4.0.6 # via pylons, sqlalchemy-migrate +decorator==4.2.1 # via pylons, sqlalchemy-migrate fanstatic==0.12 -Flask==0.11.1 Flask-Babel==0.11.2 -FormEncode==1.3.0 # via pylons +Flask==0.12.2 +FormEncode==1.3.1 # via pylons funcsigs==1.0.2 # via beaker html5lib==1.0.1 # via bleach itsdangerous==0.24 # via flask -Jinja2==2.8 # via flask -Mako==1.0.4 # via pylons +Jinja2==2.8 # via flask, flask-babel +Mako==1.0.7 # via pylons Markdown==2.6.7 -MarkupSafe==0.23 # via jinja2, mako, webhelpers +MarkupSafe==1.0 # via jinja2, mako, webhelpers nose==1.3.7 # via pylons ofs==0.4.2 Pairtree==0.7.1-T @@ -31,22 +31,22 @@ PasteScript==2.0.2 # via pylons pbr==1.10.0 # via sqlalchemy-migrate polib==1.0.7 psycopg2==2.7.3.2 -Pygments==2.1.3 # via weberror +Pygments==2.2.0 # via weberror Pylons==0.9.7 pysolr==3.6.0 python-dateutil==1.5 python-magic==0.4.12 pytz==2016.7 pyutilib.component.core==4.6.4 -redis==2.10.5 # via rq -repoze.lru==0.6 # via routes +redis==2.10.6 # via rq +repoze.lru==0.7 # via routes repoze.who-friendlyform==1.0.8 repoze.who==2.3 requests==2.11.1 Routes==1.13 # via pylons rq==0.6.0 simplejson==3.10.0 -six==1.10.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate +six==1.11.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate sqlalchemy-migrate==0.10.0 SQLAlchemy==1.1.11 # via sqlalchemy-migrate sqlparse==0.2.2 @@ -59,7 +59,7 @@ WebError==0.13.1 # via pylons WebHelpers==1.3 # via pylons WebOb==1.0.8 # via fanstatic, pylons, repoze.who, repoze.who-friendlyform, weberror, webtest WebTest==1.4.3 # via pylons -Werkzeug==0.11.10 # via flask +Werkzeug==0.14.1 # via flask zope.interface==4.3.2 # The following packages are commented out because they are