From 823f6d55272c93bb537026f02f54dbcbeb458e57 Mon Sep 17 00:00:00 2001 From: Sam Smith Date: Mon, 12 Feb 2018 13:10:19 +0000 Subject: [PATCH 01/21] Update facet_list.html Remove brackets and add some `span`s to the `facet_list` items, to offer more styling options. Note: The `hidden` and `badge` classes are not necessary to achieve the above, but are in keeping with the use of Bootstrap styling throughout CKAN. --- ckan/templates/snippets/facet_list.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckan/templates/snippets/facet_list.html b/ckan/templates/snippets/facet_list.html index 864660721ff..d5e7089aed5 100644 --- a/ckan/templates/snippets/facet_list.html +++ b/ckan/templates/snippets/facet_list.html @@ -64,10 +64,12 @@

{% set href = h.remove_url_param(name, item.name, extras=extras, alternative_url=alternative_url) if item.active else h.add_url_param(new_params={name: item.name}, extras=extras, alternative_url=alternative_url) %} {% set label = label_function(item) if label_function else item.display_name %} {% set label_truncated = h.truncate(label, 22) if not label_function else label %} - {% set count = count_label(item['count']) if count_label else ('(%d)' % item['count']) %} + {% set count = count_label(item['count']) if count_label else ('%d' % item['count']) %}
  • - {{ label_truncated }} {{ count }} + {{ label_truncated }} + + {{ count }}
    • {% endfor %} From 7acde57919161b144ac488f83b12709eada024ea Mon Sep 17 00:00:00 2001 From: cclauss Date: Tue, 13 Feb 2018 14:27:41 +0100 Subject: [PATCH 02/21] long was renamed to int in Python 3 --- ckan/lib/dictization/__init__.py | 5 +++++ ckan/tests/logic/test_validators.py | 4 ++-- ckanext/multilingual/plugin.py | 5 +++++ ckanext/stats/tests/test_stats_lib.py | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ckan/lib/dictization/__init__.py b/ckan/lib/dictization/__init__.py index 149cfa74a68..b3dec3bf19c 100644 --- a/ckan/lib/dictization/__init__.py +++ b/ckan/lib/dictization/__init__.py @@ -11,6 +11,11 @@ except AttributeError: RowProxy = sqlalchemy.engine.base.RowProxy +try: + long # Python 2 +except NameError: + long = int # Python 3 + # NOTE # The functions in this file contain very generic methods for dictizing objects diff --git a/ckan/tests/logic/test_validators.py b/ckan/tests/logic/test_validators.py index d5806953041..8a1b5594133 100644 --- a/ckan/tests/logic/test_validators.py +++ b/ckan/tests/logic/test_validators.py @@ -151,7 +151,7 @@ def test_name_validator_with_invalid_value(self): # Non-string names aren't allowed as names. 13, 23.7, - 100L, + 100, 1.0j, None, True, @@ -229,7 +229,7 @@ def test_user_name_validator_with_non_string_value(self): non_string_values = [ 13, 23.7, - 100L, + 100, 1.0j, None, True, diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py index 9981ac31a45..238aa5e7371 100644 --- a/ckanext/multilingual/plugin.py +++ b/ckanext/multilingual/plugin.py @@ -7,6 +7,11 @@ from ckan.common import request, config, c from ckan.logic import get_action +try: + long # Python 2 +except NameError: + long = int # Python 3 + def translate_data_dict(data_dict): '''Return the given dict (e.g. a dataset dict) with as many of its fields diff --git a/ckanext/stats/tests/test_stats_lib.py b/ckanext/stats/tests/test_stats_lib.py index e927afa4ae9..109ea797d99 100644 --- a/ckanext/stats/tests/test_stats_lib.py +++ b/ckanext/stats/tests/test_stats_lib.py @@ -92,7 +92,7 @@ def test_largest_groups(self): def test_top_tags(self): tags = Stats.top_tags() tags = [(tag.name, count) for tag, count in tags] - assert_equal(tags, [('tag1', 1L)]) + assert_equal(tags, [('tag1', 1)]) def test_top_package_creators(self): creators = Stats.top_package_creators() From a371ef83b7a75e1ed8f84490c331fcb398dd215d Mon Sep 17 00:00:00 2001 From: cclauss Date: Tue, 13 Feb 2018 17:19:19 +0100 Subject: [PATCH 03/21] from six.moves import xrange for Python 3 (en masse) --- ckan/lib/jinja_extensions.py | 2 ++ ckan/tests/controllers/test_api.py | 2 ++ ckan/tests/controllers/test_group.py | 2 ++ ckan/tests/logic/action/test_get.py | 2 ++ 4 files changed, 8 insertions(+) diff --git a/ckan/lib/jinja_extensions.py b/ckan/lib/jinja_extensions.py index d8545dfe307..f3c54163b55 100644 --- a/ckan/lib/jinja_extensions.py +++ b/ckan/lib/jinja_extensions.py @@ -11,6 +11,8 @@ from jinja2.utils import open_if_exists, escape from jinja2 import Environment +from six.moves import xrange + import ckan.lib.base as base import ckan.lib.helpers as h diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py index 115b9a49c49..32a05dbcb80 100644 --- a/ckan/tests/controllers/test_api.py +++ b/ckan/tests/controllers/test_api.py @@ -13,6 +13,8 @@ from nose.tools import assert_equal, assert_in, eq_ from pyfakefs import fake_filesystem +from six.moves import xrange + from ckan.lib.helpers import url_for import ckan.tests.helpers as helpers from ckan.tests import factories diff --git a/ckan/tests/controllers/test_group.py b/ckan/tests/controllers/test_group.py index f03d9dd11aa..29a3b790c03 100644 --- a/ckan/tests/controllers/test_group.py +++ b/ckan/tests/controllers/test_group.py @@ -3,6 +3,8 @@ from bs4 import BeautifulSoup from nose.tools import assert_equal, assert_true, assert_in +from six.moves import xrange + from ckan.lib.helpers import url_for import ckan.tests.helpers as helpers diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index fa556010b42..e0148287e54 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -4,6 +4,8 @@ import nose.tools +from six.moves import xrange + from ckan import __version__ import ckan.logic as logic import ckan.plugins as p From 355cb45139c6e253b623d4ccd04e57f4edb1d18f Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 10:34:11 +0000 Subject: [PATCH 04/21] [#3512] Newer beaker version fixes issue of being served from an osx (case insensitive) file system. --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index c86f1288d17..97a2353ccbe 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,7 @@ # The file contains the direct ckan requirements. # Use pip-compile to create a requirements.txt file from this Babel==2.3.4 -Beaker==1.8.1 # Needs to be pinned to a more up to date version than the Pylons one +Beaker==1.9.0 # Needs to be pinned to a more up to date version than the Pylons one bleach==2.1.2 click==6.7 fanstatic==0.12 diff --git a/requirements.txt b/requirements.txt index 468254a7d6f..9eedbfe53a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # argparse==1.4.0 # via ofs Babel==2.3.4 -Beaker==1.8.1 # via pylons +Beaker==1.9.0 # via pylons bleach==2.1.2 click==6.7 decorator==4.0.6 # via pylons, sqlalchemy-migrate From 2dc73729188899570ca5e18eba315bec9663ec85 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 12:11:47 +0000 Subject: [PATCH 05/21] Remove unhelpful docs * 'pip install' is from the internet to your venv. Your current dir is irrelevant. * personal repo tip is badly worded. Knowing how to submit a pull request involves a lot more than this and seems unhelpful to dilute these install instructions which are already really long. * ckan 2.0 pip-requirements.txt - this is so old and unsupported, we should not be documenting it any more. --- .../installing/install-from-source.rst | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index e12a974d436..d192b0cfd91 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -105,14 +105,6 @@ b. Install the recommended ``setuptools`` version: pip install setuptools==\ |min_setuptools_version| c. Install the CKAN source code into your virtualenv. - .. important:: - - For the following commands, make sure you are in your CKAN default directory. E.g. - - .. parsed-literal:: - - cd /usr/lib/ckan/default/ - To install the latest stable release of CKAN (CKAN |latest_release_version|), run: @@ -128,15 +120,6 @@ c. Install the CKAN source code into your virtualenv. pip install -e 'git+\ |git_url|\#egg=ckan' - .. tip:: - - If you would like to work submit a pull request with your changes, be sure you are working from a cloned repository. - Use your personal repository URL instead of the CKAN repository. E.g. - - .. parsed-literal:: - - pip install -e 'git=https://github.com/{your-username}/ckan.git#egg=ckan' - .. warning:: The development version may contain bugs and should not be used for @@ -145,10 +128,6 @@ c. Install the CKAN source code into your virtualenv. d. Install the Python modules that CKAN requires into your virtualenv: - .. versionchanged:: 2.1 - In CKAN 2.0 and earlier the requirement file was called - ``pip-requirements.txt`` not ``requirements.txt`` as below. - .. parsed-literal:: pip install -r |virtualenv|/src/ckan/requirements.txt @@ -380,4 +359,3 @@ This is seen occasionally with Jetty and Ubuntu 14.04. It requires a solr-jetty wget https://launchpad.net/~vshn/+archive/ubuntu/solr/+files/solr-jetty-jsp-fix_1.0.2_all.deb sudo dpkg -i solr-jetty-jsp-fix_1.0.2_all.deb sudo service jetty restart - From 20a24c23f7833ff2aad1e8bc3aa56ebfa087f9ca Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 12:37:22 +0000 Subject: [PATCH 06/21] [#4020] Move the who.ini step to be before the "paster db init" step. --- .../installing/install-from-source.rst | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index e12a974d436..72ae97d1fea 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -106,13 +106,13 @@ b. Install the recommended ``setuptools`` version: c. Install the CKAN source code into your virtualenv. .. important:: - + For the following commands, make sure you are in your CKAN default directory. E.g. - + .. parsed-literal:: - + cd /usr/lib/ckan/default/ - + To install the latest stable release of CKAN (CKAN |latest_release_version|), run: @@ -129,14 +129,14 @@ c. Install the CKAN source code into your virtualenv. pip install -e 'git+\ |git_url|\#egg=ckan' .. tip:: - + If you would like to work submit a pull request with your changes, be sure you are working from a cloned repository. Use your personal repository URL instead of the CKAN repository. E.g. - + .. parsed-literal:: - + pip install -e 'git=https://github.com/{your-username}/ckan.git#egg=ckan' - + .. warning:: The development version may contain bugs and should not be used for @@ -235,8 +235,19 @@ site_url .. _postgres-init: +---------------------- +6. Link to ``who.ini`` +---------------------- + +``who.ini`` (the Repoze.who configuration file) needs to be accessible in the +same directory as your CKAN config file, so create a symlink to it: + +.. parsed-literal:: + + ln -s |virtualenv|/src/ckan/who.ini |config_dir|/who.ini + ------------------------- -6. Create database tables +7. Create database tables ------------------------- Now that you have a configuration file that has the correct settings for your @@ -256,7 +267,7 @@ You should see ``Initialising DB: SUCCESS``. See `4. Create a CKAN config file`_. ----------------------- -7. Set up the DataStore +8. Set up the DataStore ----------------------- .. note :: @@ -268,17 +279,6 @@ Follow the instructions in :doc:`/maintaining/datastore` to create the required databases and users, set the right permissions and set the appropriate values in your CKAN config file. ----------------------- -8. Link to ``who.ini`` ----------------------- - -``who.ini`` (the Repoze.who configuration file) needs to be accessible in the -same directory as your CKAN config file, so create a symlink to it: - -.. parsed-literal:: - - ln -s |virtualenv|/src/ckan/who.ini |config_dir|/who.ini - --------------- 9. You're done! --------------- @@ -380,4 +380,3 @@ This is seen occasionally with Jetty and Ubuntu 14.04. It requires a solr-jetty wget https://launchpad.net/~vshn/+archive/ubuntu/solr/+files/solr-jetty-jsp-fix_1.0.2_all.deb sudo dpkg -i solr-jetty-jsp-fix_1.0.2_all.deb sudo service jetty restart - From 7b104e4ec1c2065759f6ceb4c969e7066d3c8c70 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 12:41:28 +0000 Subject: [PATCH 07/21] Fix whitespace in doc --- .../installing/install-from-source.rst | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/doc/maintaining/installing/install-from-source.rst b/doc/maintaining/installing/install-from-source.rst index e12a974d436..940182f9dd2 100644 --- a/doc/maintaining/installing/install-from-source.rst +++ b/doc/maintaining/installing/install-from-source.rst @@ -106,13 +106,13 @@ b. Install the recommended ``setuptools`` version: c. Install the CKAN source code into your virtualenv. .. important:: - + For the following commands, make sure you are in your CKAN default directory. E.g. - + .. parsed-literal:: - + cd /usr/lib/ckan/default/ - + To install the latest stable release of CKAN (CKAN |latest_release_version|), run: @@ -129,14 +129,14 @@ c. Install the CKAN source code into your virtualenv. pip install -e 'git+\ |git_url|\#egg=ckan' .. tip:: - + If you would like to work submit a pull request with your changes, be sure you are working from a cloned repository. Use your personal repository URL instead of the CKAN repository. E.g. - + .. parsed-literal:: - + pip install -e 'git=https://github.com/{your-username}/ckan.git#egg=ckan' - + .. warning:: The development version may contain bugs and should not be used for @@ -380,4 +380,3 @@ This is seen occasionally with Jetty and Ubuntu 14.04. It requires a solr-jetty wget https://launchpad.net/~vshn/+archive/ubuntu/solr/+files/solr-jetty-jsp-fix_1.0.2_all.deb sudo dpkg -i solr-jetty-jsp-fix_1.0.2_all.deb sudo service jetty restart - From c01d4dbbbf7e7ddd6f0235bd15ebea3e3b4e0116 Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 12:57:55 +0000 Subject: [PATCH 08/21] More occasions where ckan 2.0 is mentioned for development --- doc/contributing/documentation.rst | 4 ---- doc/contributing/test.rst | 8 -------- doc/maintaining/upgrading/upgrade-source.rst | 5 ----- 3 files changed, 17 deletions(-) diff --git a/doc/contributing/documentation.rst b/doc/contributing/documentation.rst index a600de053a4..b45f45e77c6 100644 --- a/doc/contributing/documentation.rst +++ b/doc/contributing/documentation.rst @@ -67,10 +67,6 @@ install the dependencies necessary for building CKAN. In this example we'll create a virtualenv in a folder called ``pyenv``. Run these commands in a terminal: -.. versionchanged:: 2.1 - In CKAN 2.0 and earlier the requirements file was called - ``pip-requirements-docs.txt``, not ``dev-requirements.txt`` as below. - :: virtualenv --no-site-packages pyenv diff --git a/doc/contributing/test.rst b/doc/contributing/test.rst index 20d61b74a06..b0780b7ae6f 100644 --- a/doc/contributing/test.rst +++ b/doc/contributing/test.rst @@ -32,10 +32,6 @@ virtual environment: Install nose and other test-specific CKAN dependencies into your virtual environment: -.. versionchanged:: 2.1 - In CKAN 2.0 and earlier the requirements file was called - ``pip-requirements-test.txt``, not ``dev-requirements.txt`` as below. - .. parsed-literal:: pip install -r |virtualenv|/src/ckan/dev-requirements.txt @@ -46,10 +42,6 @@ environment: Set up the test databases ~~~~~~~~~~~~~~~~~~~~~~~~~ -.. versionchanged:: 2.1 - Previously |postgres| tests used the databases defined in your - ``development.ini`` file, instead of using their own test databases. - Create test databases: .. parsed-literal:: diff --git a/doc/maintaining/upgrading/upgrade-source.rst b/doc/maintaining/upgrading/upgrade-source.rst index afdb3e2dd96..3d60ae20181 100644 --- a/doc/maintaining/upgrading/upgrade-source.rst +++ b/doc/maintaining/upgrading/upgrade-source.rst @@ -33,10 +33,6 @@ CKAN release you're upgrading to: #. Update CKAN's dependencies: - .. versionchanged:: 2.1 - In CKAN 2.0 and earlier the requirements file was called - ``pip-requirements.txt``, not ``requirements.txt`` as below. - :: pip install --upgrade -r requirements.txt @@ -84,4 +80,3 @@ CKAN release you're upgrading to: You should now be able to visit your CKAN website in your web browser and see that it's running the new version of CKAN. - From 95264ebe9c57fd31463f3fa828521854a939c89d Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 15:01:45 +0000 Subject: [PATCH 09/21] pip-compile -U puts the latest version of Beaker in requirements.txt, so it was not useful to pin it in requirements.in --- requirements.in | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.in b/requirements.in index 97a2353ccbe..c72e16c082c 100644 --- a/requirements.in +++ b/requirements.in @@ -1,7 +1,6 @@ # The file contains the direct ckan requirements. # Use pip-compile to create a requirements.txt file from this Babel==2.3.4 -Beaker==1.9.0 # Needs to be pinned to a more up to date version than the Pylons one bleach==2.1.2 click==6.7 fanstatic==0.12 From 7003fe8a4041632455ed72454cf80720485e2e0a Mon Sep 17 00:00:00 2001 From: David Read Date: Thu, 15 Feb 2018 15:10:34 +0000 Subject: [PATCH 10/21] Updated deps by running: pip-compile -U --output-file requirements.txt requirements.in --- requirements.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9eedbfe53a9..2b05f76817c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,22 +5,22 @@ # pip-compile --output-file requirements.txt requirements.in # argparse==1.4.0 # via ofs -Babel==2.3.4 +Babel==2.3.4 # via flask-babel Beaker==1.9.0 # via pylons bleach==2.1.2 click==6.7 -decorator==4.0.6 # via pylons, sqlalchemy-migrate +decorator==4.2.1 # via pylons, sqlalchemy-migrate fanstatic==0.12 -Flask==0.11.1 Flask-Babel==0.11.2 -FormEncode==1.3.0 # via pylons +Flask==0.11.1 # via flask-babel +FormEncode==1.3.1 # via pylons funcsigs==1.0.2 # via beaker html5lib==1.0.1 # via bleach itsdangerous==0.24 # via flask -Jinja2==2.8 # via flask -Mako==1.0.4 # via pylons +Jinja2==2.8 # via flask, flask-babel +Mako==1.0.7 # via pylons Markdown==2.6.7 -MarkupSafe==0.23 # via jinja2, mako, webhelpers +MarkupSafe==1.0 # via jinja2, mako, webhelpers nose==1.3.7 # via pylons ofs==0.4.2 Pairtree==0.7.1-T @@ -31,22 +31,22 @@ PasteScript==2.0.2 # via pylons pbr==1.10.0 # via sqlalchemy-migrate polib==1.0.7 psycopg2==2.7.3.2 -Pygments==2.1.3 # via weberror +Pygments==2.2.0 # via weberror Pylons==0.9.7 pysolr==3.6.0 python-dateutil==1.5 python-magic==0.4.12 pytz==2016.7 pyutilib.component.core==4.6.4 -redis==2.10.5 # via rq -repoze.lru==0.6 # via routes +redis==2.10.6 # via rq +repoze.lru==0.7 # via routes repoze.who-friendlyform==1.0.8 repoze.who==2.3 requests==2.11.1 Routes==1.13 # via pylons rq==0.6.0 simplejson==3.10.0 -six==1.10.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate +six==1.11.0 # via bleach, html5lib, pastescript, pyutilib.component.core, sqlalchemy-migrate sqlalchemy-migrate==0.10.0 SQLAlchemy==1.1.11 # via sqlalchemy-migrate sqlparse==0.2.2 @@ -59,7 +59,7 @@ WebError==0.13.1 # via pylons WebHelpers==1.3 # via pylons WebOb==1.0.8 # via fanstatic, pylons, repoze.who, repoze.who-friendlyform, weberror, webtest WebTest==1.4.3 # via pylons -Werkzeug==0.11.10 # via flask +Werkzeug==0.14.1 # via flask zope.interface==4.3.2 # The following packages are commented out because they are From 5f065b0b752606a53a5b85805af7b8c1ba954718 Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 16 Feb 2018 08:11:26 +0000 Subject: [PATCH 11/21] [#3929] Try a compromise between other options. --- ckan/migration/versions/001_add_existing_tables.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ckan/migration/versions/001_add_existing_tables.py b/ckan/migration/versions/001_add_existing_tables.py index 4335c15dacb..c346a889b45 100644 --- a/ckan/migration/versions/001_add_existing_tables.py +++ b/ckan/migration/versions/001_add_existing_tables.py @@ -2,10 +2,17 @@ from sqlalchemy import * from migrate import * +from ckan.common import config as ckan_config def upgrade(migrate_engine): - meta = MetaData() + schema = ckan_config.get(u'ckan.migrations.target_schema') or 'public' + # we specify the schema here because of a clash with another 'state' table + # in the mdillon/postgis container. You only need to change the value in the + # config if you've altered the default schema from 'public' in your + # postgresql.conf. Because this is such a rarely needed option, it is + # otherwise undocumented. + meta = MetaData(schema=schema) state = Table('state', meta, Column('id', Integer() , primary_key=True, nullable=False), From e0f1aafa8ecd52dc6e8238826ca1e40dd92a266d Mon Sep 17 00:00:00 2001 From: cclauss Date: Sat, 17 Feb 2018 13:17:55 +0100 Subject: [PATCH 12/21] Change basestring --> six.string_types for Python 3 --- ckan/controllers/group.py | 3 ++- ckan/controllers/package.py | 3 ++- ckan/lib/dictization/model_save.py | 3 ++- ckan/lib/extract.py | 3 ++- ckan/lib/helpers.py | 15 ++++++++------- ckan/lib/search/common.py | 4 +++- ckan/lib/search/query.py | 8 ++++---- ckan/logic/__init__.py | 5 +++-- ckan/logic/action/get.py | 13 +++++++------ ckan/logic/converters.py | 10 ++++++---- ckan/model/types.py | 5 +++-- ckan/plugins/core.py | 3 ++- ckan/tests/legacy/html_check.py | 10 ++++++---- ckanext/datastore/backend/postgres.py | 10 ++++++---- ckanext/datastore/helpers.py | 10 ++++++---- ckanext/datastore/logic/schema.py | 6 ++++-- ckanext/datastore/plugin.py | 10 ++++++---- ckanext/multilingual/plugin.py | 12 +++++++----- 18 files changed, 79 insertions(+), 54 deletions(-) diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index c70e4b612c9..968dce61bf4 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -5,6 +5,7 @@ from urllib import urlencode from pylons.i18n import get_lang +from six import string_types import ckan.lib.base as base import ckan.lib.helpers as h @@ -268,7 +269,7 @@ def search_url(params): controller = lookup_group_controller(group_type) action = 'bulk_process' if c.action == 'bulk_process' else 'read' url = h.url_for(controller=controller, action=action, id=id) - params = [(k, v.encode('utf-8') if isinstance(v, basestring) + params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py index 2c4a2e85914..fbb313a82f9 100644 --- a/ckan/controllers/package.py +++ b/ckan/controllers/package.py @@ -9,6 +9,7 @@ from ckan.common import config from paste.deploy.converters import asbool import paste.fileapp +from six import string_types import ckan.logic as logic import ckan.lib.base as base @@ -45,7 +46,7 @@ def _encode_params(params): - return [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) + return [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py index 60476666682..790bfe23c95 100644 --- a/ckan/lib/dictization/model_save.py +++ b/ckan/lib/dictization/model_save.py @@ -5,6 +5,7 @@ import logging from sqlalchemy.orm import class_mapper +from six import string_types import ckan.lib.dictization as d import ckan.lib.helpers as h @@ -460,7 +461,7 @@ def package_api_to_dict(api1_dict, context): for key, value in api1_dict.iteritems(): new_value = value if key == 'tags': - if isinstance(value, basestring): + if isinstance(value, string_types): new_value = [{"name": item} for item in value.split()] else: new_value = [{"name": item} for item in value] diff --git a/ckan/lib/extract.py b/ckan/lib/extract.py index 927e53f5ca2..6a57ce664ac 100644 --- a/ckan/lib/extract.py +++ b/ckan/lib/extract.py @@ -3,6 +3,7 @@ import re from jinja2.ext import babel_extract as extract_jinja2 import lib.jinja_extensions +from six import string_types jinja_extensions = ''' jinja2.ext.do, jinja2.ext.with_, @@ -24,7 +25,7 @@ def jinja2_cleaner(fileobj, *args, **kw): for lineno, func, message, finder in raw_extract: - if isinstance(message, basestring): + if isinstance(message, string_types): message = lib.jinja_extensions.regularise_html(message) elif message is not None: message = (lib.jinja_extensions.regularise_html(message[0]) diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 46f98164da7..523cc95547d 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -35,6 +35,7 @@ from flask import url_for as _flask_default_url_for from werkzeug.routing import BuildError as FlaskRouteBuildError import i18n +from six import string_types import ckan.exceptions import ckan.model as model @@ -111,7 +112,7 @@ def _datestamp_to_datetime(datetime_): :rtype: datetime ''' - if isinstance(datetime_, basestring): + if isinstance(datetime_, string_types): try: datetime_ = date_str_to_datetime(datetime_) except TypeError: @@ -171,7 +172,7 @@ def redirect_to(*args, **kw): _url = '' skip_url_parsing = False parse_url = kw.pop('parse_url', False) - if uargs and len(uargs) is 1 and isinstance(uargs[0], basestring) \ + if uargs and len(uargs) is 1 and isinstance(uargs[0], string_types) \ and (uargs[0].startswith('/') or is_url(uargs[0])) \ and parse_url is False: skip_url_parsing = True @@ -358,7 +359,7 @@ def _url_for_flask(*args, **kw): # The API routes used to require a slash on the version number, make sure # we remove it if (args and args[0].startswith('api.') and - isinstance(kw.get('ver'), basestring) and + isinstance(kw.get('ver'), string_types) and kw['ver'].startswith('/')): kw['ver'] = kw['ver'].replace('/', '') @@ -1051,7 +1052,7 @@ def get_param_int(name, default=10): def _url_with_params(url, params): if not params: return url - params = [(k, v.encode('utf-8') if isinstance(v, basestring) else str(v)) + params = [(k, v.encode('utf-8') if isinstance(v, string_types) else str(v)) for k, v in params] return url + u'?' + urlencode(params) @@ -1802,7 +1803,7 @@ def remove_url_param(key, value=None, replace=None, controller=None, instead. ''' - if isinstance(key, basestring): + if isinstance(key, string_types): keys = [key] else: keys = key @@ -2137,7 +2138,7 @@ def format_resource_items(items): # Sometimes values that can't be converted to ints can sneak # into the db. In this case, just leave them as they are. pass - elif isinstance(value, basestring): + elif isinstance(value, string_types): # check if strings are actually datetime/number etc if re.search(reg_ex_datetime, value): datetime_ = date_str_to_datetime(value) @@ -2534,7 +2535,7 @@ def get_translated(data_dict, field): return data_dict[field + u'_translated'][language] except KeyError: val = data_dict.get(field, '') - return _(val) if val and isinstance(val, basestring) else val + return _(val) if val and isinstance(val, string_types) else val @core_helper diff --git a/ckan/lib/search/common.py b/ckan/lib/search/common.py index 8d031891760..060f62f4175 100644 --- a/ckan/lib/search/common.py +++ b/ckan/lib/search/common.py @@ -5,6 +5,8 @@ import re import pysolr import simplejson +from six import string_types + log = logging.getLogger(__name__) @@ -72,7 +74,7 @@ def make_connection(decode_dates=True): def solr_datetime_decoder(d): for k, v in d.items(): - if isinstance(v, basestring): + if isinstance(v, string_types): possible_datetime = re.search(pysolr.DATETIME_REGEX, v) if possible_datetime: date_values = possible_datetime.groupdict() diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 13d75e7a799..9ad3adf0b1c 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -49,7 +49,7 @@ def convert_legacy_parameters_to_solr(legacy_params): non_solr_params = set(legacy_params.keys()) - VALID_SOLR_PARAMETERS for search_key in non_solr_params: value_obj = legacy_params[search_key] - value = value_obj.replace('+', ' ') if isinstance(value_obj, basestring) else value_obj + value = value_obj.replace('+', ' ') if isinstance(value_obj, six.string_types) else value_obj if search_key == 'all_fields': if value: solr_params['fl'] = '*' @@ -62,7 +62,7 @@ def convert_legacy_parameters_to_solr(legacy_params): elif search_key == 'tags': if isinstance(value_obj, list): tag_list = value_obj - elif isinstance(value_obj, basestring): + elif isinstance(value_obj, six.string_types): tag_list = [value_obj] else: raise SearchQueryError('Was expecting either a string or JSON list for the tags parameter: %r' % value) @@ -173,7 +173,7 @@ def run(self, query=None, fields=None, options=None, **kwargs): else: options.update(kwargs) - if isinstance(query, basestring): + if isinstance(query, six.string_types): query = [query] query = query[:] # don't alter caller's query list. @@ -220,7 +220,7 @@ def run(self, fields={}, options=None, **kwargs): # action. query = [] for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, six.string_types): terms = terms.split() for term in terms: query.append(':'.join([field, term])) diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py index e5dee92206e..fdbf527806e 100644 --- a/ckan/logic/__init__.py +++ b/ckan/logic/__init__.py @@ -7,6 +7,7 @@ from collections import defaultdict import formencode.validators +from six import string_types import ckan.model as model import ckan.authz as authz @@ -175,7 +176,7 @@ def clean_dict(data_dict): if not isinstance(value, list): continue for inner_dict in value[:]: - if isinstance(inner_dict, basestring): + if isinstance(inner_dict, string_types): break if not any(inner_dict.values()): value.remove(inner_dict) @@ -516,7 +517,7 @@ def get_or_bust(data_dict, keys): not in the given dictionary ''' - if isinstance(keys, basestring): + if isinstance(keys, string_types): keys = [keys] import ckan.logic.schema as schema diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index d68d3b310d2..a2c9e0fcbf9 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -11,6 +11,7 @@ from ckan.common import config import sqlalchemy from paste.deploy.converters import asbool +from six import string_types import ckan.lib.dictization import ckan.logic as logic @@ -2109,7 +2110,7 @@ def resource_search(context, data_dict): {'fields': _('Do not specify if using "query" parameter')}) elif query is not None: - if isinstance(query, basestring): + if isinstance(query, string_types): query = [query] try: fields = dict(pair.split(":", 1) for pair in query) @@ -2125,7 +2126,7 @@ def resource_search(context, data_dict): # So maintain that behaviour split_terms = {} for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = terms.split() split_terms[field] = terms fields = split_terms @@ -2143,7 +2144,7 @@ def resource_search(context, data_dict): resource_fields = model.Resource.get_columns() for field, terms in fields.items(): - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] if field not in resource_fields: @@ -2215,7 +2216,7 @@ def _tag_search(context, data_dict): model = context['model'] terms = data_dict.get('query') or data_dict.get('q') or [] - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] terms = [t.strip() for t in terms if t.strip()] @@ -2402,7 +2403,7 @@ def term_translation_show(context, data_dict): # This action accepts `terms` as either a list of strings, or a single # string. terms = _get_or_bust(data_dict, 'terms') - if isinstance(terms, basestring): + if isinstance(terms, string_types): terms = [terms] if terms: q = q.where(trans_table.c.term.in_(terms)) @@ -2411,7 +2412,7 @@ def term_translation_show(context, data_dict): # string. if 'lang_codes' in data_dict: lang_codes = _get_or_bust(data_dict, 'lang_codes') - if isinstance(lang_codes, basestring): + if isinstance(lang_codes, string_types): lang_codes = [lang_codes] q = q.where(trans_table.c.lang_code.in_(lang_codes)) diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py index c747555e910..ed97f67de73 100644 --- a/ckan/logic/converters.py +++ b/ckan/logic/converters.py @@ -2,6 +2,8 @@ import json +from six import string_types + import ckan.model as model import ckan.lib.navl.dictization_functions as df import ckan.logic.validators as validators @@ -60,7 +62,7 @@ def callable(key, data, errors, context): new_tags = data.get(key) if not new_tags: return - if isinstance(new_tags, basestring): + if isinstance(new_tags, string_types): new_tags = [new_tags] # get current number of tags @@ -173,7 +175,7 @@ def convert_group_name_or_id_to_id(group_name_or_id, context): def convert_to_json_if_string(value, context): - if isinstance(value, basestring): + if isinstance(value, string_types): try: return json.loads(value) except ValueError: @@ -183,13 +185,13 @@ def convert_to_json_if_string(value, context): def convert_to_list_if_string(value, context=None): - if isinstance(value, basestring): + if isinstance(value, string_types): return [value] else: return value def remove_whitespace(value, context): - if isinstance(value, basestring): + if isinstance(value, string_types): return value.strip() return value diff --git a/ckan/model/types.py b/ckan/model/types.py index b05fcf335eb..c4351730126 100644 --- a/ckan/model/types.py +++ b/ckan/model/types.py @@ -7,6 +7,7 @@ import simplejson as json from sqlalchemy import types +from six import string_types import meta @@ -74,7 +75,7 @@ def process_bind_param(self, value, engine): if value is None or value == {}: # ensure we stores nulls in db not json "null" return None else: - if isinstance(value, basestring): + if isinstance(value, string_types): return unicode(value) else: return unicode(json.dumps(value, ensure_ascii=False)) @@ -89,7 +90,7 @@ def iso_date_to_datetime_for_sqlite(datetime_or_iso_date_if_sqlite): # to call this to convert it into a datetime type. When running on # postgres then you have a datetime anyway, so this function doesn't # do anything. - if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, basestring): + if meta.engine_is_sqlite() and isinstance(datetime_or_iso_date_if_sqlite, string_types): return datetime.datetime.strptime(datetime_or_iso_date_if_sqlite, '%Y-%m-%d %H:%M:%S.%f') else: diff --git a/ckan/plugins/core.py b/ckan/plugins/core.py index 07456fc8803..14098016d0d 100644 --- a/ckan/plugins/core.py +++ b/ckan/plugins/core.py @@ -12,6 +12,7 @@ from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin from pyutilib.component.core import Plugin as _pca_Plugin from paste.deploy.converters import asbool +from six import string_types import interfaces @@ -244,7 +245,7 @@ def _get_service(plugin_name): :return: the service object ''' - if isinstance(plugin_name, basestring): + if isinstance(plugin_name, string_types): for group in GROUPS: iterator = iter_entry_points( group=group, diff --git a/ckan/tests/legacy/html_check.py b/ckan/tests/legacy/html_check.py index 5b549942ee6..e8bc05e502b 100644 --- a/ckan/tests/legacy/html_check.py +++ b/ckan/tests/legacy/html_check.py @@ -3,13 +3,15 @@ import re import sgmllib +from six import string_types + import paste.fixture class HtmlCheckMethods(object): '''A collection of methods to check properties of a html page, usually in the form returned by paster.''' - + def named_div(self, div_name, html): 'strips html to just the
    section' the_html = self._get_html_from_res(html) @@ -31,15 +33,15 @@ def sidebar(self, html): def strip_tags(self, res): '''Call strip_tags on a TestResponse object to strip any and all HTML and normalise whitespace.''' - if not isinstance(res, basestring): + if not isinstance(res, string_types): res = res.body.decode('utf-8') - return Stripper().strip(res) + return Stripper().strip(res) def check_named_element(self, html, tag_name, *html_to_find): '''Searches in the html and returns True if it can find a particular tag and all its subtags & data which contains all the of the html_to_find''' - named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?)' % {'tag':tag_name}) + named_element_re = re.compile('(<(%(tag)s\w*).*?(>.*?)' % {'tag':tag_name}) html_str = self._get_html_from_res(html) self._check_html(named_element_re, html_str.replace('\n', ''), html_to_find) diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py index c0c287a9c21..7c4118dc5b8 100644 --- a/ckanext/datastore/backend/postgres.py +++ b/ckanext/datastore/backend/postgres.py @@ -15,6 +15,8 @@ import json from cStringIO import StringIO +from six import string_types + import ckan.lib.cli as cli import ckan.plugins as p import ckan.plugins.toolkit as toolkit @@ -374,7 +376,7 @@ def _where_clauses(data_dict, fields_types): # add full-text search where clause q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): ts_query_alias = _ts_query_alias() clause_str = u'_full_text @@ {0}'.format(ts_query_alias) clauses.append((clause_str,)) @@ -409,7 +411,7 @@ def _textsearch_query(data_dict): statements = [] rank_columns = [] plain = data_dict.get('plain', True) - if isinstance(q, basestring): + if isinstance(q, string_types): query, rank = _build_query_and_rank_statements( lang, q, plain) statements.append(query) @@ -469,7 +471,7 @@ def _sort(data_dict, fields_types): if not sort: q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): return [_ts_rank_alias()] elif isinstance(q, dict): return [_ts_rank_alias(field) for field in q @@ -1196,7 +1198,7 @@ def validate(context, data_dict): for key, values in data_dict_copy.iteritems(): if not values: continue - if isinstance(values, basestring): + if isinstance(values, string_types): value = values elif isinstance(values, (list, tuple)): value = values[0] diff --git a/ckanext/datastore/helpers.py b/ckanext/datastore/helpers.py index b616f0f9405..523bb096774 100644 --- a/ckanext/datastore/helpers.py +++ b/ckanext/datastore/helpers.py @@ -6,6 +6,8 @@ import paste.deploy.converters as converters import sqlparse +from six import string_types + from ckan.plugins.toolkit import get_action, ObjectNotFound, NotAuthorized log = logging.getLogger(__name__) @@ -57,10 +59,10 @@ def validate_int(i, non_negative=False): return i >= 0 or not non_negative -def _strip(input): - if isinstance(input, basestring) and len(input) and input[0] == input[-1]: - return input.strip().strip('"') - return input +def _strip(s): + if isinstance(s, string_types) and len(s) and s[0] == s[-1]: + return s.strip().strip('"') + return s def should_fts_index_field_type(field_type): diff --git a/ckanext/datastore/logic/schema.py b/ckanext/datastore/logic/schema.py index 903e016d71c..9a7d5795925 100644 --- a/ckanext/datastore/logic/schema.py +++ b/ckanext/datastore/logic/schema.py @@ -2,6 +2,8 @@ import json +from six import string_types + import ckan.plugins as p import ckan.lib.navl.dictization_functions as df @@ -48,13 +50,13 @@ def list_of_strings_or_lists(key, data, errors, context): if not isinstance(value, list): raise df.Invalid('Not a list') for x in value: - if not isinstance(x, basestring) and not isinstance(x, list): + if not isinstance(x, string_types) and not isinstance(x, list): raise df.Invalid('%s: %s' % ('Neither a string nor a list', x)) def list_of_strings_or_string(key, data, errors, context): value = data.get(key) - if isinstance(value, basestring): + if isinstance(value, string_types): return list_of_strings_or_lists(key, data, errors, context) diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index f47cbbfd796..8d0c6b7b584 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -2,6 +2,7 @@ import logging +from six import string_types import ckan.plugins as p import ckan.logic as logic @@ -213,16 +214,17 @@ def datastore_validate(self, context, data_dict, fields_types): q = data_dict.get('q') if q: - if isinstance(q, basestring): + if isinstance(q, string_types): del data_dict['q'] elif isinstance(q, dict): for key in q.keys(): - if key in fields_types and isinstance(q[key], basestring): + if key in fields_types and isinstance(q[key], + string_types): del q[key] language = data_dict.get('language') if language: - if isinstance(language, basestring): + if isinstance(language, string_types): del data_dict['language'] plain = data_dict.get('plain') @@ -249,7 +251,7 @@ def datastore_validate(self, context, data_dict, fields_types): if limit: is_positive_int = datastore_helpers.validate_int(limit, non_negative=True) - is_all = isinstance(limit, basestring) and limit.lower() == 'all' + is_all = isinstance(limit, string_types) and limit.lower() == 'all' if is_positive_int or is_all: del data_dict['limit'] diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py index 238aa5e7371..c4615151572 100644 --- a/ckanext/multilingual/plugin.py +++ b/ckanext/multilingual/plugin.py @@ -1,5 +1,7 @@ # encoding: utf-8 +from six import string_types + import ckan from ckan.plugins import SingletonPlugin, implements, IPackageController from ckan.plugins import IGroupController, IOrganizationController, ITagController, IResourceController @@ -31,7 +33,7 @@ def translate_data_dict(data_dict): for (key, value) in flattened.items(): if value in (None, True, False): continue - elif isinstance(value, basestring): + elif isinstance(value, string_types): terms.add(value) elif isinstance(value, (int, long)): continue @@ -79,7 +81,7 @@ def translate_data_dict(data_dict): # Don't try to translate values that aren't strings. translated_flattened[key] = value - elif isinstance(value, basestring): + elif isinstance(value, string_types): if value in desired_translations: translated_flattened[key] = desired_translations[value] else: @@ -127,7 +129,7 @@ def translate_resource_data_dict(data_dict): for (key, value) in flattened.items(): if value in (None, True, False): continue - elif isinstance(value, basestring): + elif isinstance(value, string_types): terms.add(value) elif isinstance(value, (int, long)): continue @@ -170,7 +172,7 @@ def translate_resource_data_dict(data_dict): # Don't try to translate values that aren't strings. translated_flattened[key] = value - elif isinstance(value, basestring): + elif isinstance(value, string_types): if value in desired_translations: translated_flattened[key] = desired_translations[value] else: @@ -229,7 +231,7 @@ def before_index(self, search_data): if not isinstance(value, list): value = [value] for item in value: - if isinstance(item, basestring): + if isinstance(item, string_types): all_terms.append(item) field_translations = get_action('term_translation_show')( From ec795ab4d20078247e35216dda7d7840907f27f6 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 17:42:24 -0500 Subject: [PATCH 13/21] [#3989] inline docs in dictionary_form.html --- .../datastore/snippets/dictionary_form.html | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html index fad1d662df7..41266c9aae9 100644 --- a/ckanext/datastore/templates/datastore/snippets/dictionary_form.html +++ b/ckanext/datastore/templates/datastore/snippets/dictionary_form.html @@ -1,16 +1,25 @@ {% import 'macros/form.html' as form %}

    {{ _( "Field {num}.").format(num=position) }} {{ field.id }} ({{ field.type }})

    - {{ form.select('info__' ~ position ~ '__type_override', - label=_('Type Override'), options=[ - {'name': '', 'value': ''}, - {'name': 'text', 'value': 'text'}, - {'name': 'numeric', 'value': 'numeric'}, - {'name': 'timestamp', 'value': 'timestamp'}, - ], selected=field.get('info', {}).get('type_override', '')) }} - {{ form.input('info__' ~ position ~ '__label', - label=_('Label'), id='field-f' ~ position ~ 'label', - value=field.get('info', {}).get('label', ''), classes=['control-full']) }} - {{ form.markdown('info__' ~ position ~ '__notes', - label=_('Description'), id='field-d' ~ position ~ 'notes', - value=field.get('info', {}).get('notes', '')) }} + +{# + Data Dictionary fields may be added this snippet. New fields following + the 'info__' ~ position ~ '__namegoeshere' convention will be saved + as part of the "info" object on the column. +#} + +{{ form.select('info__' ~ position ~ '__type_override', + label=_('Type Override'), options=[ + {'name': '', 'value': ''}, + {'name': 'text', 'value': 'text'}, + {'name': 'numeric', 'value': 'numeric'}, + {'name': 'timestamp', 'value': 'timestamp'}, + ], selected=field.get('info', {}).get('type_override', '')) }} + +{{ form.input('info__' ~ position ~ '__label', + label=_('Label'), id='field-f' ~ position ~ 'label', + value=field.get('info', {}).get('label', ''), classes=['control-full']) }} + +{{ form.markdown('info__' ~ position ~ '__notes', + label=_('Description'), id='field-d' ~ position ~ 'notes', + value=field.get('info', {}).get('notes', '')) }} From 699856611f1f82b4bc4f580abc8f85e0408f728e Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 19:33:39 -0500 Subject: [PATCH 14/21] [#3989] document field 'info' object --- doc/maintaining/datastore.rst | 42 ++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index c4da16b5d16..03d7627aaad 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -304,22 +304,40 @@ Fields Fields define the column names and the type of the data in a column. A field is defined as follows:: { - "id": # a string which defines the column name + "id": # the column name (required) "type": # the data type for the column + "info": { + "label": # human-readable label for column + "notes": # markdown description of column + "type_override": # type for datapusher to use when importing data + ...: # other user-defined fields + } } -Field **types are optional** and will be guessed by the DataStore from the provided data. However, setting the types ensures that future inserts will not fail because of wrong types. See :ref:`valid-types` for details on which types are valid. +Field types not provided will be guessed based on the first row of provided data. +Set the types to ensure that future inserts will not fail because of an incorrectly +guessed type. See :ref:`valid-types` for details on which types are valid. + +Extra ``"info"`` field values will be stored along with the column. ``"label"``, +``"notes"`` and ``"type_override"`` can be managed from the default Data Dictionary +form. Additional fields can be stored by customizing the Data Dictionary form or by +passing their values to the API directly. Example:: [ { - "id": "foo", - "type": "int4" + "id": "code_number", + "type": "numeric" }, { - "id": "bar" - # type is optional + "id": "description" + "type": "text", + "info": { + "label": "Description", + "notes": "A brief usage description for this code", + "example": "Used for temporary service interruptions" + } } ] @@ -331,19 +349,21 @@ Records A record is the data to be inserted in a DataStore resource and is defined as follows:: { - "": # data to be set - # .. more data + column_1_id: value_1, + columd_2_id: value_2, + ... } Example:: [ { - "foo": 100, - "bar": "Here's some text" + "code_number": 10, + "description": "Submitted successfully" }, { - "foo": 42 + "code_number": 42, + "description": "In progress" } ] From eba5abbdc71f5c2d555c958507bbe5718246eb69 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 19:40:54 -0500 Subject: [PATCH 15/21] [#3989] promote "Download resource" and "Extending DataStore" to top level sections --- doc/maintaining/datastore.rst | 40 +++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index 03d7627aaad..80bffd22bc3 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -233,6 +233,25 @@ alongside CKAN. To install this please look at the docs here: http://docs.ckan.org/projects/datapusher +.. _dump: + +--------------------- +Downloading resources +--------------------- + +A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. + +For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``. + +Other formats supported include tab-separated values (``?format=tsv``), +JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible +tab-separated file use +``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``. + +.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values + + + ----------------- The DataStore API ----------------- @@ -279,23 +298,6 @@ API reference :members: -.. _dump: - -Download resource ------------------ - -A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. - -For an Excel-compatible CSV file use ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?bom=true``. - -Other formats supported include tab-separated values (``?format=tsv``), -JSON (``?format=json``) and XML (``?format=xml``). E.g. to download an Excel-compatible -tab-separated file use -``{CKAN-URL}/datastore/dump/{RESOURCE-ID}?format=tsv&bom=true``. - -.. _CSV: https://en.wikipedia.org/wiki/Comma-separated_values - - .. _fields: Fields @@ -457,8 +459,10 @@ name oid The PostgreSQL object ID of the table that belongs to name. + +------------------- Extending DataStore -=================== +------------------- Starting from CKAN version 2.7, backend used in DataStore can be replaced with custom one. For this purpose, custom extension must implement `ckanext.datastore.interfaces.IDatastoreBackend`, which provides one method - `register_backends`. It should return dictonary with names of custom backends as keys and classes, that represent those backends as values. Each class supposed to be inherited from `ckanext.datastore.backend.DatastoreBackend`. From 297e4295eb61d645d9a6e1d83de13846d447bc05 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 20:07:12 -0500 Subject: [PATCH 16/21] [#3989] document Data Dictionary form --- doc/maintaining/datastore.rst | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index 80bffd22bc3..0fe2db7de2b 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -233,10 +233,34 @@ alongside CKAN. To install this please look at the docs here: http://docs.ckan.org/projects/datapusher +.. _data_dictionary: + +--------------- +Data Dictionary +--------------- + +DataStore columns may be described with a Data Dictionary. A Data Dictionary tab +will appear when editing any resource with a DataStore table. +The Data Dictionary form allows entering the following values for +each column: + +1. Type Override: the type to be used the next time DataPusher is run to load + data into this column +2. Label: a human-friendly label for this column +3. Description: a full description for this column in markdown format + +Extension developers may add new fields to this form by overriding the default +Data Dictionary form template ``datastore/snippets/dictionary_form.html``. + +The Data Dictionary is set through the API as part of the :ref:`fields` passed +to :meth:`~ckanext.datastore.logic.action.datastore_create` and +returned from :meth:`~ckanext.datastore.logic.action.datastore_search`. + + .. _dump: --------------------- -Downloading resources +Downloading Resources --------------------- A DataStore resource can be downloaded in the `CSV`_ file format from ``{CKAN-URL}/datastore/dump/{RESOURCE-ID}``. From 43139d636c08d4a63c6c8deabda4a3c77de24b57 Mon Sep 17 00:00:00 2001 From: Ian Ward Date: Sun, 18 Feb 2018 20:12:26 -0500 Subject: [PATCH 17/21] [#3989] link back from Fields to Data Dictionary --- doc/maintaining/datastore.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/maintaining/datastore.rst b/doc/maintaining/datastore.rst index 0fe2db7de2b..9d058b49c60 100644 --- a/doc/maintaining/datastore.rst +++ b/doc/maintaining/datastore.rst @@ -244,10 +244,10 @@ will appear when editing any resource with a DataStore table. The Data Dictionary form allows entering the following values for each column: -1. Type Override: the type to be used the next time DataPusher is run to load - data into this column -2. Label: a human-friendly label for this column -3. Description: a full description for this column in markdown format +* **Type Override:** the type to be used the next time DataPusher is run to load + data into this column +* **Label:** a human-friendly label for this column +* **Description:** a full description for this column in markdown format Extension developers may add new fields to this form by overriding the default Data Dictionary form template ``datastore/snippets/dictionary_form.html``. @@ -345,7 +345,7 @@ Set the types to ensure that future inserts will not fail because of an incorrec guessed type. See :ref:`valid-types` for details on which types are valid. Extra ``"info"`` field values will be stored along with the column. ``"label"``, -``"notes"`` and ``"type_override"`` can be managed from the default Data Dictionary +``"notes"`` and ``"type_override"`` can be managed from the default :ref:`data_dictionary` form. Additional fields can be stored by customizing the Data Dictionary form or by passing their values to the API directly. From d3c91dae94e5586abe63a8db5bcd6fff7d0fb4fe Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 10:17:38 +0100 Subject: [PATCH 18/21] Upgrade Solr Docker image Bump to patch release to include security fixes --- contrib/docker/solr/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index baa0acc1899..801a3963ac9 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -1,4 +1,4 @@ -FROM solr:6.2 +FROM solr:6.2.2 MAINTAINER Open Knowledge # Enviroment From fb982dea8c4279968d6ba586e7eae25720a9c00c Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 10:27:49 +0100 Subject: [PATCH 19/21] Fix wrong version --- contrib/docker/solr/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index 801a3963ac9..16c6aee0feb 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -1,4 +1,4 @@ -FROM solr:6.2.2 +FROM solr:6.6.2 MAINTAINER Open Knowledge # Enviroment From 272b59e4eb953bc5656ebbba54d66307a74a49c9 Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 10:35:51 +0100 Subject: [PATCH 20/21] Upgrade scripts version --- contrib/docker/solr/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index 16c6aee0feb..9dc8c2063f6 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -14,11 +14,11 @@ RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data # Adding Files ADD ./contrib/docker/solr/solrconfig.xml \ ./ckan/config/solr/schema.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/currency.xml \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ -https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.0.0/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/currency.xml \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/synonyms.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/stopwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/basic_configs/conf/protwords.txt \ +https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/solr/server/solr/configsets/data_driven_schema_configs/conf/elevate.xml \ /opt/solr/server/solr/$SOLR_CORE/conf/ # Create Core.properties From 49ba9f3ad455933a4bed0d29377ae7db0a1a506a Mon Sep 17 00:00:00 2001 From: amercader Date: Thu, 22 Feb 2018 11:22:20 +0100 Subject: [PATCH 21/21] Run as Solr user, as this version won't start as root by default --- contrib/docker/solr/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/docker/solr/Dockerfile b/contrib/docker/solr/Dockerfile index 9dc8c2063f6..59ce3492f42 100644 --- a/contrib/docker/solr/Dockerfile +++ b/contrib/docker/solr/Dockerfile @@ -4,9 +4,6 @@ MAINTAINER Open Knowledge # Enviroment ENV SOLR_CORE ckan -# User -USER root - # Create Directories RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/conf RUN mkdir -p /opt/solr/server/solr/$SOLR_CORE/data @@ -25,4 +22,9 @@ https://raw.githubusercontent.com/apache/lucene-solr/releases/lucene-solr/6.6.2/ RUN echo name=$SOLR_CORE > /opt/solr/server/solr/$SOLR_CORE/core.properties # Giving ownership to Solr + +USER root RUN chown -R $SOLR_USER:$SOLR_USER /opt/solr/server/solr/$SOLR_CORE + +# User +USER $SOLR_USER:$SOLR_USER