Merge branch 'master' into 5108-allow-extensions-define-click-cmds

ckan/cli/datastore.py

@@ -6,7 +6,7 @@
 
 import click
 
-from ckan.cli import error_shout
+from ckan.model import parse_db_config
 from ckan.common import config
 
 import ckanext.datastore as datastore_module
@@ -30,9 +30,9 @@ def set_permissions():
     u'''Emit an SQL script that will set the permissions for the datastore
     users as configured in your configuration file.'''
 
-    write_url = parse_db_config(u'ckan.datastore.write_url')
-    read_url = parse_db_config(u'ckan.datastore.read_url')
-    db_url = parse_db_config(u'sqlalchemy.url')
+    write_url = _parse_db_config(u'ckan.datastore.write_url')
+    read_url = _parse_db_config(u'ckan.datastore.read_url')
+    db_url = _parse_db_config(u'sqlalchemy.url')
 
     # Basic validation that read and write URLs reference the same database.
     # This obviously doesn't check they're the same database (the hosts/ports
@@ -102,25 +102,13 @@ def dump(ctx, resource_id, output_file, format, offset, limit, bom):
         )
 
 
-def parse_db_config(config_key=u'sqlalchemy.url'):
-    u''' Takes a config key for a database connection url and parses it into
-    a dictionary. Expects a url like:
-
-    'postgres://tester:pass@localhost/ckantest3'
-    '''
-    url = config[config_key]
-    regex = [
-        u'^\\s*(?P<db_type>\\w*)', u'://', u'(?P<db_user>[^:]*)', u':?',
-        u'(?P<db_pass>[^@]*)', u'@', u'(?P<db_host>[^/:]*)', u':?',
-        u'(?P<db_port>[^/]*)', u'/', u'(?P<db_name>[\\w.-]*)'
-    ]
-    db_details_match = re.match(u''.join(regex), url)
-    if not db_details_match:
+def _parse_db_config(config_key=u'sqlalchemy.url'):
+    db_config = parse_db_config(config_key)
+    if not db_config:
         click.secho(
-            u'Could not extract db details from url: %r' % url,
+            u'Could not extract db details from url: %r' % config[config_key],
             fg=u'red',
             bold=True
         )
         raise click.Abort()
-    db_details = db_details_match.groupdict()
-    return db_details
+    return db_config

ckan/common.py

@@ -124,34 +124,40 @@ def clear(self):
             flask.current_app.config.clear()
         except RuntimeError:
             pass
-        try:
-            pylons.config.clear()
-            # Pylons set this default itself
-            pylons.config[u'lang'] = None
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                pylons.config.clear()
+                # Pylons set this default itself
+                pylons.config[u'lang'] = None
+            except TypeError:
+                pass
 
     def __setitem__(self, key, value):
         self.store[key] = value
         try:
             flask.current_app.config[key] = value
         except RuntimeError:
             pass
-        try:
-            pylons.config[key] = value
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                pylons.config[key] = value
+            except TypeError:
+                pass
 
     def __delitem__(self, key):
         del self.store[key]
         try:
             del flask.current_app.config[key]
         except RuntimeError:
             pass
-        try:
-            del pylons.config[key]
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                del pylons.config[key]
+            except TypeError:
+                pass
 
 
 def _get_request():

ckan/config/environment.py

@@ -6,8 +6,8 @@
 import warnings
 import pytz
 
+import six
 import sqlalchemy
-from pylons import config as pylons_config
 
 from six.moves.urllib.parse import urlparse
 
@@ -20,7 +20,6 @@
 from ckan.lib.redis import is_redis_available
 import ckan.lib.render as render
 import ckan.lib.search as search
-import ckan.lib.plugins as lib_plugins
 import ckan.logic as logic
 import ckan.authz as authz
 import ckan.lib.jinja_extensions as jinja_extensions
@@ -30,6 +29,10 @@
 from ckan.common import _, ungettext, config
 from ckan.exceptions import CkanConfigurationException
 
+if six.PY2:
+    from pylons import config as pylons_config
+
+
 log = logging.getLogger(__name__)
 
 
@@ -42,35 +45,36 @@ def load_environment(global_conf, app_conf):
     Configure the Pylons environment via the ``pylons.config`` object. This
     code should only need to be run once.
     """
-    # this must be run at a time when the env is semi-setup, thus inlined here.
-    # Required by the deliverance plugin and iATI
-    from pylons.wsgiapp import PylonsApp
-    import pkg_resources
-    find_controller_generic = getattr(
-        PylonsApp.find_controller,
-        '_old_find_controller',
-        PylonsApp.find_controller)
-
-    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
-    def find_controller(self, controller):
-        if controller in self.controller_classes:
-            return self.controller_classes[controller]
-        # Check to see if its a dotted name
-        if '.' in controller or ':' in controller:
-            ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
-
-            if hasattr(ep, 'resolve'):
-                # setuptools >= 10.2
-                mycontroller = ep.resolve()
-            else:
-                # setuptools >= 11.3
-                mycontroller = ep.load(False)
-
-            self.controller_classes[controller] = mycontroller
-            return mycontroller
-        return find_controller_generic(self, controller)
-    find_controller._old_find_controller = find_controller_generic
-    PylonsApp.find_controller = find_controller
+    if six.PY2:
+        # this must be run at a time when the env is semi-setup, thus inlined
+        # here. Required by the deliverance plugin and iATI
+        from pylons.wsgiapp import PylonsApp
+        import pkg_resources
+        find_controller_generic = getattr(
+            PylonsApp.find_controller,
+            '_old_find_controller',
+            PylonsApp.find_controller)
+
+        # This is from pylons 1.0 source, will monkey-patch into 0.9.7
+        def find_controller(self, controller):
+            if controller in self.controller_classes:
+                return self.controller_classes[controller]
+            # Check to see if its a dotted name
+            if '.' in controller or ':' in controller:
+                ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
+
+                if hasattr(ep, 'resolve'):
+                    # setuptools >= 10.2
+                    mycontroller = ep.resolve()
+                else:
+                    # setuptools >= 11.3
+                    mycontroller = ep.load(False)
+
+                self.controller_classes[controller] = mycontroller
+                return mycontroller
+            return find_controller_generic(self, controller)
+        find_controller._old_find_controller = find_controller_generic
+        PylonsApp.find_controller = find_controller
 
     os.environ['CKAN_CONFIG'] = global_conf['__file__']
 
@@ -97,13 +101,15 @@ def find_controller(self, controller):
     config.update(global_conf)
     config.update(app_conf)
 
-    # Initialize Pylons own config object
-    pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths)
+    if six.PY2:
+        # Initialize Pylons own config object
+        pylons_config.init_app(
+            global_conf, app_conf, package='ckan', paths=paths)
 
-    # Update the main CKAN config object with the Pylons specific stuff, as it
-    # is quite hard to keep them separated. This should be removed once Pylons
-    # support is dropped
-    config.update(pylons_config)
+        # Update the main CKAN config object with the Pylons specific stuff,
+        # as it is quite hard to keep them separated. This should be removed
+        # once Pylons support is dropped
+        config.update(pylons_config)
 
     # Setup the SQLAlchemy database engine
     # Suppress a couple of sqlalchemy warnings
@@ -230,20 +236,23 @@ def update_config():
                              config.get('solr_password'))
     search.check_solr_schema_version()
 
-    routes_map = routing.make_map()
+    if six.PY2:
+        routes_map = routing.make_map()
 
     lib_plugins.reset_package_plugins()
     lib_plugins.register_package_plugins()
     lib_plugins.reset_group_plugins()
     lib_plugins.register_group_plugins()
 
-    config['routes.map'] = routes_map
-    # The RoutesMiddleware needs its mapper updating if it exists
-    if 'routes.middleware' in config:
-        config['routes.middleware'].mapper = routes_map
-    # routes.named_routes is a CKAN thing
-    config['routes.named_routes'] = routing.named_routes
-    config['pylons.app_globals'] = app_globals.app_globals
+    if six.PY2:
+        config['routes.map'] = routes_map
+        # The RoutesMiddleware needs its mapper updating if it exists
+        if 'routes.middleware' in config:
+            config['routes.middleware'].mapper = routes_map
+        # routes.named_routes is a CKAN thing
+        config['routes.named_routes'] = routing.named_routes
+        config['pylons.app_globals'] = app_globals.app_globals
+
     # initialise the globals
     app_globals.app_globals._init()
 
@@ -276,13 +285,14 @@ def update_config():
     # root logger.
     logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
 
-    # Create Jinja2 environment
-    env = jinja_extensions.Environment(
-        **jinja_extensions.get_jinja_env_options())
-    env.install_gettext_callables(_, ungettext, newstyle=True)
-    # custom filters
-    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
-    config['pylons.app_globals'].jinja_env = env
+    if six.PY2:
+        # Create Jinja2 environment
+        env = jinja_extensions.Environment(
+            **jinja_extensions.get_jinja_env_options())
+        env.install_gettext_callables(_, ungettext, newstyle=True)
+        # custom filters
+        env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
+        config['pylons.app_globals'].jinja_env = env
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)

ckan/config/middleware/__init__.py

@@ -2,43 +2,44 @@
 
 """WSGI app initialization"""
 
-import webob
-from routes import request_config as routes_request_config
+import logging
 
 import six
 from six.moves.urllib.parse import urlparse, quote
 
 from ckan.lib.i18n import get_locales_from_config
 from ckan.config.environment import load_environment
 from ckan.config.middleware.flask_app import make_flask_stack
-from ckan.config.middleware.pylons_app import make_pylons_stack
 from ckan.common import config
 
-import logging
-log = logging.getLogger(__name__)
 
-# This monkey-patches the webob request object because of the way it messes
-# with the WSGI environ.
+log = logging.getLogger(__name__)
 
-# Start of webob.requests.BaseRequest monkey patch
-original_charset__set = webob.request.BaseRequest._charset__set
+if six.PY2:
+    import webob
+    from routes import request_config as routes_request_config
+    from ckan.config.middleware.pylons_app import make_pylons_stack
 
+    # This monkey-patches the webob request object because of the way it messes
+    # with the WSGI environ.
 
-def custom_charset__set(self, charset):
-    original_charset__set(self, charset)
-    if self.environ.get('CONTENT_TYPE', '').startswith(';'):
-        self.environ['CONTENT_TYPE'] = ''
+    # Start of webob.requests.BaseRequest monkey patch
+    original_charset__set = webob.request.BaseRequest._charset__set
 
+    def custom_charset__set(self, charset):
+        original_charset__set(self, charset)
+        if self.environ.get('CONTENT_TYPE', '').startswith(';'):
+            self.environ['CONTENT_TYPE'] = ''
 
-webob.request.BaseRequest._charset__set = custom_charset__set
+    webob.request.BaseRequest._charset__set = custom_charset__set
 
-webob.request.BaseRequest.charset = property(
-    webob.request.BaseRequest._charset__get,
-    custom_charset__set,
-    webob.request.BaseRequest._charset__del,
-    webob.request.BaseRequest._charset__get.__doc__)
+    webob.request.BaseRequest.charset = property(
+        webob.request.BaseRequest._charset__get,
+        custom_charset__set,
+        webob.request.BaseRequest._charset__del,
+        webob.request.BaseRequest._charset__get.__doc__)
 
-# End of webob.requests.BaseRequest monkey patch
+    # End of webob.requests.BaseRequest monkey patch
 
 # This is a test Flask request context to be used internally.
 # Do not use it!
@@ -53,12 +54,16 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
 
     load_environment(conf, app_conf)
 
-    pylons_app = make_pylons_stack(conf, full_stack, static_files,
-                                   **app_conf)
     flask_app = make_flask_stack(conf, **app_conf)
-
-    app = AskAppDispatcherMiddleware({'pylons_app': pylons_app,
-                                      'flask_app': flask_app})
+    if six.PY2:
+        pylons_app = make_pylons_stack(
+            conf, full_stack, static_files, **app_conf)
+
+        app = AskAppDispatcherMiddleware(
+            {'pylons_app': pylons_app,
+             'flask_app': flask_app})
+    else:
+        app = flask_app
 
     # Set this internal test request context with the configured environment so
     # it can be used when calling url_for from tests