Merge remote-tracking branch 'upstream/master' into datastore-dump-ui

CHANGELOG.rst

@@ -21,6 +21,20 @@ Note: This version requires re-running the 'datastore set-permissions' command
    CKAN developers should also re-run set-permissions on the test database:
    :ref:`datastore-test-set-permissions`
 
+v2.7.2 2017-09-28
+=================
+
+ * Include missing minified JavaScript files
+
+v2.7.1 2017-09-27
+=================
+
+ * add field_name to image_upload macro when uploading resources (#3766)
+ * Add some missing major changes to change log. (#3799)
+ * _mail_recipient header override (#3781)
+ * skip url parsing in redirect (#3499)
+ * Fix multiple errors in i18n of JS modules (#3590)
+ * Standardize on url_for on popup (#3831)
 
 v2.7.0 2017-08-02
 =================
@@ -33,8 +47,7 @@ General notes:
    configuration option.
  * This version requires a requirements upgrade on source installations
  * This version requires a database upgrade
- * This version does not require a Solr schema upgrade (You may want to
-   upgrade the schema if you want to target Solr>=5, see #2914)
+ * This version requires a Solr schema upgrade
  * There are several old features being officially deprecated starting from
    this version. Check the *Deprecations* section to be prepared.
 
@@ -126,6 +139,13 @@ Deprecations:
    jobs (http://docs.ckan.org/en/latest/maintaining/background-tasks.html). Extensions can still
    of course use Celery but they will need to handle the management themselves.
 
+v2.6.4 2017-09-27
+=================
+
+* Mail recepient header override (#3781)
+* Skip url parsing in redirect (#3499)
+* Support non root for fanstatic (#3618)
+
 v2.6.3 2017-08-02
 =================
 
@@ -268,6 +288,13 @@ Bug fixes:
 API changes and deprecations:
  * Replace `c.__version__` with new helper `h.ckan_version()` (`#3103 <https://github.com/ckan/ckan/pull/3103>`_)
 
+v2.5.7 2017-09-27
+=================
+
+* Allow overriding email headers (#3781)
+* Support non-root instances on fanstatic (#3618)
+* Add missing close button on organization page (#3814)
+
 v2.5.6 2017-08-02
 =================
 
@@ -412,20 +439,12 @@ v2.5.0 2015-12-17
 
 Cancelled release
 
-v2.4.6 2017-02-22
+v2.4.9 2017-09-27
 =================
 
- * Use the url_for() helper for datapusher URLs (#2866)
- * Resource creation date use datetime.utcnow() (#3447)
- * Fix locale error when using fix ckan.root_path
- * `render_markdown` breaks links with ampersands
- * Check group name and id during package creation
- * Use utcnow() on dashboard_mark_activities_old (#3373)
- * Fix encoding error on DataStore exception
- * Datastore doesn't add site_url to resource created via API (#3189)
- * Fix memberships after user deletion (#3265)
- * Remove idle database connection (#3260)
- * Fix package_owner_org_update action when called via the API (#2661)
+* Allow overriding email headers (#3781)
+* Support non-root instances on fanstatic (#3618)
+* Add missing close button on organization page (#3814)
 
 v2.4.8 2017-08-02
 =================

README.rst

@@ -35,10 +35,11 @@ See the `CKAN Documentation <http://docs.ckan.org>`_ for installation instructio
 
 Support
 -------
+For general discussion around CKAN, you can write to the `Google Group`_.
 
-If you need help with CKAN or want to ask a question about CKAN, use either the
-`ckan-discuss`_ mailing list or the `CKAN tag on Stack Overflow`_ (try
-searching the Stack Overflow and ckan-discuss archives for an answer to your
+If you need help with CKAN and and want to ask a or want to ask a question about CKAN, use either the
+`ckan-dev`_ mailing list or the `CKAN tag on Stack Overflow`_ (try
+searching the Stack Overflow and ckan-dev `archives`_ for an answer to your
 question first).
 
 If you've found a bug in CKAN, open a new issue on CKAN's `GitHub Issues`_ (try
@@ -47,9 +48,10 @@ searching first to see if there's already an issue for your bug).
 If you find a potential security vulnerability please email security@ckan.org,
 rather than creating a public issue on GitHub.
 
-
+.. _Google Group: https://groups.google.com/forum/#!forum/ckan-global-user-group
 .. _CKAN tag on Stack Overflow: http://stackoverflow.com/questions/tagged/ckan
-.. _ckan-discuss: http://lists.okfn.org/mailman/listinfo/ckan-discuss
+.. _ckan-dev: https://lists.okfn.org/mailman/listinfo/ckan-dev
+.. _archives: https://www.google.com/search?q=%22%5Bckan-dev%5D%22+site%3Alists.okfn.org.
 .. _GitHub Issues: https://github.com/ckan/ckan/issues
 
 

ckan/__init__.py

@@ -4,17 +4,17 @@
 
 __description__ = 'CKAN Software'
 __long_description__ = \
-'''CKAN software provides a hub for datasets. The flagship site running CKAN
-is theDataHub.org but it is also used for dozens of other open data websites
-run by governments, agencies and citizens.
+'''
+CKAN is the world's leading Open Source data portal platform.
+
+It powers dozens of Open Data portals around the world, including
+data.gov, open.canada.ca and europeandataportal.eu but also regional,
+research and community organizations.
 
-CKAN provides a place to search for open knowledge resources as well as
-register your own - be that a set of Shakespeare's works, a global
-population density database, the voting records of MPs, or 30 years of
-US patents.
+It makes easy to publish, share and find data online and is fully
+customizable via extensions and plugins.
 
-Those familiar with Freshmeat or CPAN can think of CKAN as providing an
-analogous service for open data and knowledge.
+Check https://ckan.org to know more.
 '''
 __license__ = 'AGPL'
 

ckan/config/environment.py

@@ -251,7 +251,7 @@ def update_config():
     if extra_template_paths:
         # must be first for them to override defaults
         template_paths = extra_template_paths.split(',') + template_paths
-    config['pylons.app_globals'].template_paths = template_paths
+    config['computed_template_paths'] = template_paths
 
     # Set the default language for validation messages from formencode
     # to what is set as the default locale in the config

ckan/config/middleware/__init__.py

@@ -3,10 +3,12 @@
 """WSGI app initialization"""
 import urllib
 import urlparse
+import urllib
 
 import webob
 from routes import request_config as routes_request_config
 
+from ckan.lib.i18n import get_locales_from_config
 from ckan.config.environment import load_environment
 from ckan.config.middleware.flask_app import make_flask_stack
 from ckan.config.middleware.pylons_app import make_pylons_stack

ckan/config/middleware/flask_app.py

@@ -6,6 +6,8 @@
 import itertools
 import pkgutil
 
+from jinja2 import ChoiceLoader
+
 from flask import Flask, Blueprint
 from flask.ctx import _AppCtxGlobals
 from flask.sessions import SessionInterface
@@ -18,10 +20,14 @@
 from beaker.middleware import SessionMiddleware
 from paste.deploy.converters import asbool
 from fanstatic import Fanstatic
+from repoze.who.config import WhoConfig
+from repoze.who.middleware import PluggableAuthenticationMiddleware
 
+import ckan.model as model
 from ckan.lib import helpers
 from ckan.lib import jinja_extensions
-from ckan.common import config, g, request
+from ckan.lib.render import CkanextTemplateLoader
+from ckan.common import config, g, request, ungettext
 import ckan.lib.app_globals as app_globals
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import IBlueprint, IMiddleware
@@ -51,6 +57,10 @@ def make_flask_stack(conf, **app_conf):
     app.template_folder = os.path.join(root, 'templates')
     app.app_ctx_globals_class = CKAN_AppCtxGlobals
     app.url_rule_class = CKAN_Rule
+    app.jinja_loader = ChoiceLoader([
+        app.jinja_loader,
+        CkanextTemplateLoader()
+    ])
 
     # Update Flask config with the CKAN values. We use the common config
     # object as values might have been modified on `load_environment`
@@ -120,22 +130,21 @@ def save_session(self, app, session, response):
     app.context_processor(helper_functions)
     app.context_processor(c_object)
 
+    @app.context_processor
+    def ungettext_alias():
+        u'''
+        Provide `ungettext` as an alias of `ngettext` for backwards
+        compatibility
+        '''
+        return dict(ungettext=ungettext)
+
     # Babel
     app.config[u'BABEL_TRANSLATION_DIRECTORIES'] = os.path.join(root, u'i18n')
     app.config[u'BABEL_DOMAIN'] = 'ckan'
 
     babel = Babel(app)
 
-    @babel.localeselector
-    def get_locale():
-        u'''
-        Return the value of the `CKAN_LANG` key of the WSGI environ,
-        set by the I18nMiddleware based on the URL.
-        If no value is defined, it defaults to `ckan.locale_default` or `en`.
-        '''
-        return request.environ.get(
-            u'CKAN_LANG',
-            config.get(u'ckan.locale_default', u'en'))
+    babel.localeselector(get_locale)
 
     @app.route('/hello', methods=['GET'])
     def hello_world():
@@ -188,6 +197,23 @@ def hello_world_post():
                          'make_error_log_middleware.'
                          .format(plugin.__class__.__name__))
 
+    # Initialize repoze.who
+    who_parser = WhoConfig(conf['here'])
+    who_parser.parse(open(app_conf['who.config_file']))
+
+    app = PluggableAuthenticationMiddleware(
+        app,
+        who_parser.identifiers,
+        who_parser.authenticators,
+        who_parser.challengers,
+        who_parser.mdproviders,
+        who_parser.request_classifier,
+        who_parser.challenge_decider,
+        logging.getLogger('repoze.who'),
+        logging.WARN,  # ignored
+        who_parser.remote_user_key
+    )
+
     # Update the main CKAN config object with the Flask specific keys
     # that were set here or autogenerated
     flask_config_keys = set(flask_app.config.keys()) - set(config.keys())
@@ -200,6 +226,17 @@ def hello_world_post():
     return app
 
 
+def get_locale():
+    u'''
+    Return the value of the `CKAN_LANG` key of the WSGI environ,
+    set by the I18nMiddleware based on the URL.
+    If no value is defined, it defaults to `ckan.locale_default` or `en`.
+    '''
+    return request.environ.get(
+        u'CKAN_LANG',
+        config.get(u'ckan.locale_default', u'en'))
+
+
 def ckan_before_request():
     u'''Common handler executed before all Flask requests'''
 
@@ -214,6 +251,9 @@ def ckan_before_request():
 def ckan_after_request(response):
     u'''Common handler executed after all Flask requests'''
 
+    # Dispose of the SQLALchemy session
+    model.Session.remove()
+
     # Check session cookie
     response = check_session_cookie(response)
 
@@ -322,7 +362,9 @@ def _register_core_blueprints(app):
     def is_blueprint(mm):
         return isinstance(mm, Blueprint)
 
-    for loader, name, _ in pkgutil.iter_modules(['ckan/views'], 'ckan.views.'):
+    path = os.path.join(os.path.dirname(__file__), '..', '..', 'views')
+
+    for loader, name, _ in pkgutil.iter_modules([path], 'ckan.views.'):
         module = loader.find_module(name).load_module(name)
         for blueprint in inspect.getmembers(module, is_blueprint):
             app.register_blueprint(blueprint[1])

ckan/config/routing.py

@@ -111,11 +111,8 @@ def make_map():
         if not hasattr(route, '_ckan_core'):
             route._ckan_core = False
 
-    map.connect('invite', '/__invite__/', controller='partyline', action='join_party')
-
     map.connect('home', '/', controller='home', action='index')
     map.connect('about', '/about', controller='home', action='about')
-
     # CKAN API versioned.
     register_list = [
         'package',
@@ -131,12 +128,6 @@ def make_map():
     ]
     register_list_str = '|'.join(register_list)
 
-    # /api ver 3 or none
-    with SubMapper(map, controller='api', path_prefix='/api{ver:/3|}',
-                   ver='/3') as m:
-        m.connect('/action/{logic_function}', action='action',
-                  conditions=GET_POST)
-
     # /api ver 1, 2, 3 or none
     with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|/3|}',
                    ver='/1') as m:
@@ -145,9 +136,7 @@ def make_map():
     # /api ver 1, 2 or none
     with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
                    ver='/1') as m:
-        m.connect('/tag_counts', action='tag_counts')
         m.connect('/rest', action='index')
-        m.connect('/qos/throughput/', action='throughput', conditions=GET)
 
     # /api/rest ver 1, 2 or none
     with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
@@ -173,31 +162,14 @@ def make_map():
         m.connect('/rest/{register}/{id}/:subregister/{id2}', action='delete',
                   conditions=DELETE)
 
+
     # /api/util ver 1, 2 or none
     with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}',
                    ver='/1') as m:
-        m.connect('/util/user/autocomplete', action='user_autocomplete')
-        m.connect('/util/is_slug_valid', action='is_slug_valid',
-                  conditions=GET)
-        m.connect('/util/dataset/autocomplete', action='dataset_autocomplete',
-                  conditions=GET)
-        m.connect('/util/tag/autocomplete', action='tag_autocomplete',
-                  conditions=GET)
-        m.connect('/util/resource/format_autocomplete',
-                  action='format_autocomplete', conditions=GET)
-        m.connect('/util/resource/format_icon',
-                  action='format_icon', conditions=GET)
-        m.connect('/util/group/autocomplete', action='group_autocomplete')
-        m.connect('/util/organization/autocomplete', action='organization_autocomplete',
-                  conditions=GET)
-        m.connect('/util/markdown', action='markdown')
         m.connect('/util/dataset/munge_name', action='munge_package_name')
         m.connect('/util/dataset/munge_title_to_name',
                   action='munge_title_to_package_name')
         m.connect('/util/tag/munge', action='munge_tag')
-        m.connect('/util/status', action='status')
-        m.connect('/util/snippet/{snippet_path:.*}', action='snippet')
-        m.connect('/i18n/{lang}', action='i18n_js_translations')
 
     ###########
     ## /END API
@@ -211,7 +183,7 @@ def make_map():
     with SubMapper(map, controller='package') as m:
         m.connect('search', '/dataset', action='search',
                   highlight_actions='index search')
-        m.connect('add dataset', '/dataset/new', action='new')
+        m.connect('dataset_new', '/dataset/new', action='new')
         m.connect('/dataset/{action}',
                   requirements=dict(action='|'.join([
                       'list',

ckan/config/solr/schema.xml

@@ -24,7 +24,7 @@
 <!-- We update the version when there is a backward-incompatible change to this
 schema. In this case the version should be set to the next CKAN version number.
 (x.y but not x.y.z since it needs to be a float) -->
-<schema name="ckan" version="2.7">
+<schema name="ckan" version="2.8">
 
 <types>
     <fieldType name="string" class="solr.StrField" sortMissingLast="true" omitNorms="true"/>
@@ -112,7 +112,7 @@ schema. In this case the version should be set to the next CKAN version number.
     <field name="organization" type="string" indexed="true" stored="true" multiValued="false"/>
 
     <field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
-    <field name="permission_labels" type="text" indexed="true" stored="false" multiValued="true"/>
+    <field name="permission_labels" type="string" indexed="true" stored="false" multiValued="true"/>
 
     <field name="res_name" type="textgen" indexed="true" stored="true" multiValued="true" />
     <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>

ckan/config/supervisor-ckan-worker.conf

@@ -2,7 +2,7 @@
 ; Supervisor configuration for CKAN background job worker
 ; =======================================================
 
-; 1. Copy this file to /etc/supervisr/conf.d
+; 1. Copy this file to /etc/supervisor/conf.d
 ; 2. Make sure the paths below match your setup