Merge branch 'master' into 4382-full-url-stored

.circleci/config.yml

@@ -0,0 +1,102 @@
+version: 2
+jobs:
+  test:
+    docker:
+      - image: python:2
+        environment:
+          CKAN_DATASTORE_POSTGRES_DB: datastore_test
+          CKAN_DATASTORE_POSTGRES_READ_USER: datastore_read
+          CKAN_DATASTORE_POSTGRES_READ_PWD: pass
+          CKAN_DATASTORE_POSTGRES_WRITE_USER: datastore_write
+          CKAN_DATASTORE_POSTGRES_WRITE_PWD: pass
+          CKAN_POSTGRES_DB: ckan_test
+          CKAN_POSTGRES_USER: ckan_default
+          CKAN_POSTGRES_PWD: pass
+          PGPASSWORD: ckan
+          NODE_TESTS_CONTAINER: 2
+          NOSETEST_COMMON_OPTIONS: -v --ckan --reset-db --with-pylons=test-core-circle-ci.ini --nologcapture --with-coverage --cover-package=ckan --cover-package=ckanext --with-xunit --xunit-file=/root/junit/junit.xml ckan ckanext
+      - image: postgres:10
+        environment:
+          POSTGRES_USER: ckan
+          POSTGRES_PASSWORD: ckan
+        name: ckan-postgres
+      - image: redis:3
+        name: ckan-redis
+
+    parallelism: 4
+
+    steps:
+      - checkout
+
+      - run: |
+          # SO Dependencies
+          apt update
+          case $CIRCLE_NODE_INDEX in
+            $NODE_TESTS_CONTAINER)
+            curl -sL https://deb.nodesource.com/setup_10.x | bash -
+            apt install -y nodejs
+            npm install mocha-phantomjs@3.5.0 phantomjs@~1.9.1
+            ;;
+          esac
+          apt install -y postgresql-client solr-jetty openjdk-8-jdk
+
+          # Python Dependencies
+          pip install -r requirement-setuptools.txt
+          pip install -r requirements.txt
+          pip install -r dev-requirements.txt
+          python setup.py develop
+
+          # SOLR config
+          cp ~/project/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
+          service jetty9 restart || true  # erroring out but does seem to work
+
+          # Database Creation
+          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_POSTGRES_USER} WITH PASSWORD '${CKAN_POSTGRES_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_POSTGRES_USER} ${CKAN_POSTGRES_DB}
+          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_READ_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_READ_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          psql --host=ckan-postgres --username=ckan --command="CREATE USER ${CKAN_DATASTORE_POSTGRES_WRITE_USER} WITH PASSWORD '${CKAN_DATASTORE_POSTGRES_WRITE_PWD}' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
+          createdb --encoding=utf-8 --host=ckan-postgres --username=ckan --owner=${CKAN_DATASTORE_POSTGRES_WRITE_USER} ${CKAN_DATASTORE_POSTGRES_DB}
+
+          # Database Initialization
+          paster datastore -c test-core-circle-ci.ini set-permissions | psql --host=ckan-postgres --username=ckan
+          paster db init -c test-core-circle-ci.ini
+
+      # Tests Backend, split across containers by segments
+      - run: |
+          mkdir -p ~/junit
+          case $CIRCLE_NODE_INDEX in
+            0) nosetests $NOSETEST_COMMON_OPTIONS --segments 0123
+            ;;
+            1) nosetests $NOSETEST_COMMON_OPTIONS --segments 4567
+            ;;
+            2) nosetests $NOSETEST_COMMON_OPTIONS --segments 89ab
+            ;;
+            3) nosetests $NOSETEST_COMMON_OPTIONS --segments cdef
+            ;;
+          esac
+      - store_test_results:
+          path: ~/junit
+
+      # Tests Frontend, only in one container
+      - run:
+          command: |
+            case $CIRCLE_NODE_INDEX in
+              $NODE_TESTS_CONTAINER) paster serve test-core-circle-ci.ini
+              ;;
+            esac
+          background: true
+      - run:
+          command: |
+            case $CIRCLE_NODE_INDEX in
+              $NODE_TESTS_CONTAINER)
+                sleep 5
+                /root/project/node_modules/mocha-phantomjs/bin/mocha-phantomjs \
+                  http://localhost:5000/base/test/index.html
+              ;;
+            esac
+
+workflows:
+  version: 2
+  build_and_test:
+    jobs:
+      - test

CHANGELOG.rst

@@ -10,6 +10,16 @@ Changelog
 v.2.9.0 TBA
 ==================
 
+Minor changes:
+
+ * For navl schemas, the 'default' validator no longer applies the default when
+   the value is False, 0, [] or {} (#4448)
+
+Bugfixes:
+
+ * Action function "datastore_search" would calculate the total, even if you
+   set include_total=False (#4448)
+
 Deprecations:
  * ``c.action`` and ``c.controller`` variables should be avoided.
    ``ckan.plugins.toolkit.get_endpoint`` can be used instead. This function
@@ -172,8 +182,6 @@ Changes and deprecations:
  * The old Celery based background jobs have been removed in CKAN 2.8 in favour of the new RQ based
    jobs (http://docs.ckan.org/en/latest/maintaining/background-tasks.html). Extensions can still
    of course use Celery but they will need to handle the management themselves.
- * `ckan.recaptcha.version` config option is removed, since v2 is the only valid
-    version now (#4061)
  * After introducing dataset blueprint, `h.get_facet_items_dict` takes search_facets as second argument.
    This change is aimed to reduce usage of global variables in context. For a while, it has default value
    of None, in which case, `c.search_facets` will be used. But all template designers are strongly advised

Dockerfile

@@ -1,5 +1,5 @@
 # See CKAN docs on installation from Docker Compose on usage
-FROM debian:jessie
+FROM debian:stretch
 MAINTAINER Open Knowledge
 
 # Install required system packages

README.rst

@@ -20,6 +20,10 @@ CKAN: The Open Source Data Portal Software
     :target: https://coveralls.io/github/ckan/ckan?branch=master
     :alt: Coverage Status
 
+.. image:: https://badges.gitter.im/gitterHQ/gitter.svg
+    :target: https://gitter.im/ckan/chat
+    :alt: Chat on Gitter
+
 **CKAN is the world’s leading open-source data portal platform**.
 CKAN makes it easy to publish, share and work with data. It's a data management
 system that provides a powerful platform for cataloging, storing and accessing
@@ -36,7 +40,7 @@ See the `CKAN Documentation <http://docs.ckan.org>`_ for installation instructio
 Support
 -------
 If you need help with CKAN or want to ask a question, use either the
-`ckan-dev`_ mailing list or the `CKAN tag on Stack Overflow`_ (try
+`ckan-dev`_ mailing list, the `CKAN chat on Gitter`_, or the `CKAN tag on Stack Overflow`_ (try
 searching the Stack Overflow and ckan-dev `archives`_ for an answer to your
 question first).
 
@@ -47,9 +51,9 @@ If you find a potential security vulnerability please email security@ckan.org,
 rather than creating a public issue on GitHub.
 
 .. _CKAN tag on Stack Overflow: http://stackoverflow.com/questions/tagged/ckan
-.. _ckan-dev: https://lists.okfn.org/mailman/listinfo/ckan-dev
 .. _archives: https://www.google.com/search?q=%22%5Bckan-dev%5D%22+site%3Alists.okfn.org.
 .. _GitHub Issues: https://github.com/ckan/ckan/issues
+.. _CKAN chat on Gitter: https://gitter.im/ckan/chat
 
 
 Contributing to CKAN
@@ -58,17 +62,33 @@ Contributing to CKAN
 For contributing to CKAN or its documentation, see
 `CONTRIBUTING <https://github.com/ckan/ckan/blob/master/CONTRIBUTING.rst>`_.
 
-If you want to talk about CKAN development say hi to the CKAN developers on the
-`ckan-dev`_ mailing list, in the `#ckan`_ IRC channel, or on `BotBot`_.
+Mailing List
+~~~~~~~~~~~~
+
+Subscribe to the `ckan-dev`_ mailing list to receive news about upcoming releases and
+future plans as well as questions and discussions about CKAN development, deployment, etc.
+
+Community Chat
+~~~~~~~~~~~~~~
+
+If you want to talk about CKAN development say hi to the CKAN developers and members of
+the CKAN community on the public `CKAN chat on Gitter`_. Gitter is free and open-source;
+you can sign in with your GitHub, GitLab, or Twitter account.
+
+The logs for the old `#ckan`_ IRC channel (2014 to 2018) can be found here:
+https://github.com/ckan/irc-logs.
+
+Wiki
+~~~~
 
 If you've figured out how to do something with CKAN and want to document it for
-others, make a new page on the `CKAN wiki`_, and tell us about it on
-ckan-dev mailing list.
+others, make a new page on the `CKAN wiki`_ and tell us about it on the
+ckan-dev mailing list or on Gitter.
 
 .. _ckan-dev: http://lists.okfn.org/mailman/listinfo/ckan-dev
 .. _#ckan: http://webchat.freenode.net/?channels=ckan
 .. _CKAN Wiki: https://github.com/ckan/ckan/wiki
-.. _BotBot: https://botbot.me/freenode/ckan/
+.. _CKAN chat on Gitter: https://gitter.im/ckan/chat
 
 
 Copying and License

ckan/config/environment.py

@@ -45,7 +45,10 @@ def load_environment(global_conf, app_conf):
     # Required by the deliverance plugin and iATI
     from pylons.wsgiapp import PylonsApp
     import pkg_resources
-    find_controller_generic = PylonsApp.find_controller
+    find_controller_generic = getattr(
+        PylonsApp.find_controller,
+        '_old_find_controller',
+        PylonsApp.find_controller)
 
     # This is from pylons 1.0 source, will monkey-patch into 0.9.7
     def find_controller(self, controller):
@@ -65,6 +68,7 @@ def find_controller(self, controller):
             self.controller_classes[controller] = mycontroller
             return mycontroller
         return find_controller_generic(self, controller)
+    find_controller._old_find_controller = find_controller_generic
     PylonsApp.find_controller = find_controller
 
     os.environ['CKAN_CONFIG'] = global_conf['__file__']
@@ -226,9 +230,9 @@ def update_config():
     routes_map = routing.make_map()
 
     lib_plugins.reset_package_plugins()
-    lib_plugins.set_default_package_plugin()
+    lib_plugins.register_package_plugins()
     lib_plugins.reset_group_plugins()
-    lib_plugins.set_default_group_plugin()
+    lib_plugins.register_group_plugins()
 
     config['routes.map'] = routes_map
     # The RoutesMiddleware needs its mapper updating if it exists

ckan/config/middleware/flask_app.py

@@ -180,10 +180,14 @@ def hello_world_post():
     # Set up each IBlueprint extension as a Flask Blueprint
     for plugin in PluginImplementations(IBlueprint):
         if hasattr(plugin, 'get_blueprint'):
-            app.register_extension_blueprint(plugin.get_blueprint())
+            plugin_blueprints = plugin.get_blueprint()
+            if not isinstance(plugin_blueprints, list):
+                plugin_blueprints = [plugin_blueprints]
+            for blueprint in plugin_blueprints:
+                app.register_extension_blueprint(blueprint)
 
-    lib_plugins.register_group_plugins(app)
-    lib_plugins.register_package_plugins(app)
+    lib_plugins.register_package_blueprints(app)
+    lib_plugins.register_group_blueprints(app)
 
     # Set flask routes in named_routes
     for rule in app.url_map.iter_rules():
@@ -206,13 +210,16 @@ def hello_world_post():
         app = plugin.make_middleware(app, config)
 
     # Fanstatic
+    fanstatic_enable_rollup = asbool(app_conf.get('fanstatic_enable_rollup',
+                                                  False))
     if debug:
         fanstatic_config = {
             'versioning': True,
             'recompute_hashes': True,
             'minified': False,
             'bottom': True,
             'bundle': False,
+            'rollup': fanstatic_enable_rollup,
         }
     else:
         fanstatic_config = {
@@ -221,6 +228,7 @@ def hello_world_post():
             'minified': True,
             'bottom': True,
             'bundle': True,
+            'rollup': fanstatic_enable_rollup,
         }
     root_path = config.get('ckan.root_path', None)
     if root_path:
@@ -308,7 +316,8 @@ def ckan_after_request(response):
 
 def helper_functions():
     u'''Make helper functions (`h`) available to Flask templates'''
-    helpers.load_plugin_helpers()
+    if not helpers.helper_functions:
+        helpers.load_plugin_helpers()
     return dict(h=helpers.helper_functions)
 
 

ckan/config/middleware/pylons_app.py

@@ -75,13 +75,16 @@ def make_pylons_stack(conf, full_stack=True, static_files=True,
                                     cleanup_pylons_response_string)
 
     # Fanstatic
+    fanstatic_enable_rollup = asbool(app_conf.get('fanstatic_enable_rollup',
+                                                  False))
     if asbool(config.get('debug', False)):
         fanstatic_config = {
             'versioning': True,
             'recompute_hashes': True,
             'minified': False,
             'bottom': True,
             'bundle': False,
+            'rollup': fanstatic_enable_rollup,
         }
     else:
         fanstatic_config = {
@@ -90,6 +93,7 @@ def make_pylons_stack(conf, full_stack=True, static_files=True,
             'minified': True,
             'bottom': True,
             'bundle': True,
+            'rollup': fanstatic_enable_rollup,
         }
     root_path = config.get('ckan.root_path', None)
     if root_path: