Merge pull request #5130 from ckan/py3-general-support-3

[#4801] PY3 General Support - Part 3

ckan/authz.py

@@ -3,7 +3,7 @@
 import functools
 import sys
 
-from collections import defaultdict
+from collections import defaultdict, OrderedDict
 from logging import getLogger
 
 import six
@@ -13,7 +13,7 @@
 
 import ckan.plugins as p
 import ckan.model as model
-from ckan.common import OrderedDict, _, c
+from ckan.common import _, c
 
 import ckan.lib.maintain as maintain
 

ckan/cli/__init__.py

@@ -22,10 +22,13 @@ def error_shout(exception):
 )
 
 
-def load_config(config=None):
+def load_config(ini_path=None):
     from paste.deploy import appconfig
-    if config:
-        filename = os.path.abspath(config)
+
+    if ini_path:
+        if ini_path.startswith(u'~'):
+            ini_path = os.path.expanduser(ini_path)
+        filename = os.path.abspath(ini_path)
         config_source = u'-c parameter'
     elif os.environ.get(u'CKAN_INI'):
         filename = os.environ.get(u'CKAN_INI')

ckan/common.py

@@ -28,11 +28,6 @@
 
 current_app = flask.current_app
 
-try:
-    from collections import OrderedDict  # from python 2.7
-except ImportError:
-    from sqlalchemy.util import OrderedDict
-
 
 def is_flask_request():
     u'''
@@ -71,10 +66,7 @@ def streaming_response(
 
 
 def ugettext(*args, **kwargs):
-    if is_flask_request():
-        return flask_ugettext(*args, **kwargs)
-    else:
-        return pylons_ugettext(*args, **kwargs)
+    return flask_ugettext(*args, **kwargs)
 
 
 _ = ugettext

ckan/config/middleware/flask_app.py

@@ -8,9 +8,10 @@
 import itertools
 import pkgutil
 
-from flask import Flask, Blueprint, send_from_directory
+from flask import Blueprint, send_from_directory
 from flask.ctx import _AppCtxGlobals
 from flask.sessions import SessionInterface
+from flask_multistatic import MultiStaticFlask
 
 import six
 from werkzeug.exceptions import default_exceptions, HTTPException
@@ -82,7 +83,14 @@ def make_flask_stack(conf, **app_conf):
 
     debug = asbool(conf.get('debug', conf.get('DEBUG', False)))
     testing = asbool(app_conf.get('testing', app_conf.get('TESTING', False)))
-    app = flask_app = CKANFlask(__name__)
+    app = flask_app = CKANFlask(__name__, static_url_path='')
+
+    # Static files folders (core and extensions)
+    public_folder = config.get(u'ckan.base_public_folder')
+    app.static_folder = config.get(
+        'extra_public_paths', ''
+    ).split(',') + [os.path.join(root, public_folder)]
+
     app.jinja_options = jinja_extensions.get_jinja_env_options()
 
     app.debug = debug
@@ -374,7 +382,7 @@ def __getattr__(self, name):
         return getattr(app_globals.app_globals, name)
 
 
-class CKANFlask(Flask):
+class CKANFlask(MultiStaticFlask):
 
     '''Extend the Flask class with a special method called on incoming
      requests by AskAppDispatcherMiddleware.
@@ -456,9 +464,11 @@ def _register_error_handler(app):
     def error_handler(e):
         log.error(e, exc_info=sys.exc_info)
         if isinstance(e, HTTPException):
-            extra_vars = {u'code': [e.code], u'content': e.description}
-            # TODO: Remove
-            g.code = [e.code]
+            extra_vars = {
+                u'code': e.code,
+                u'content': e.description,
+                u'name': e.name
+            }
 
             return base.render(
                 u'error_document_template.html', extra_vars), e.code

ckan/config/routing.py

@@ -134,9 +134,6 @@ def make_map():
     # users
     map.redirect('/users/{url:.*}', '/user/{url}')
 
-    # robots.txt
-    map.connect('/(robots.txt)', controller='template', action='view')
-
     # Mark all unmarked routes added up until now as core routes
     for route in map.matchlist:
         if not hasattr(route, '_ckan_core'):
@@ -156,6 +153,5 @@ def make_map():
     map.redirect('/favicon.ico', config.get('ckan.favicon'))
 
     map.redirect('/*(url)/', '/{url}', _redirect_code='301 Moved Permanently')
-    map.connect('/*url', controller='template', action='view', ckan_core=True)
 
     return map

ckan/controllers/error.py

@@ -45,7 +45,7 @@ def document(self):
             cgi.escape(request.GET.get('message', ''))
         prefix = request.environ.get('SCRIPT_NAME', ''),
         code = cgi.escape(request.GET.get('code',
-                          str(original_response.status_int))),
+                          str(original_response.status_int)))
         extra_vars = {'code': code, 'content': content, 'prefix': prefix}
         return render('error_document_template.html', extra_vars=extra_vars)
 

ckan/lib/helpers.py

@@ -277,7 +277,11 @@ def _get_auto_flask_context():
     if _internal_test_request_context:
         return _internal_test_request_context
 
-    if six.Py2:
+    from ckan.tests.pytest_ckan.ckan_setup import _tests_test_request_context
+    if _tests_test_request_context:
+        return _tests_test_request_context
+
+    if six.PY2:
 
         from ckan.lib.cli import _cli_test_request_context
 
@@ -340,7 +344,6 @@ def url_for(*args, **kw):
             raise Exception('API URLs must specify the version (eg ver=3)')
 
     _auto_flask_context = _get_auto_flask_context()
-
     try:
         if _auto_flask_context:
             _auto_flask_context.push()
@@ -352,9 +355,11 @@ def url_for(*args, **kw):
         my_url = _url_for_flask(*args, **kw)
 
     except FlaskRouteBuildError:
-
-        # If it doesn't succeed, fallback to the Pylons router
-        my_url = _url_for_pylons(*args, **kw)
+        if six.PY2:
+            # If it doesn't succeed, fallback to the Pylons router
+            my_url = _url_for_pylons(*args, **kw)
+        else:
+            raise
     finally:
         if _auto_flask_context:
             _auto_flask_context.pop()
@@ -1720,22 +1725,6 @@ def tzname(self, dt):
         return None
 
 
-@core_helper
-@maintain.deprecated('h.time_ago_in_words_from_str is deprecated in 2.2 '
-                     'and will be removed.  Please use '
-                     'h.time_ago_from_timestamp instead')
-def time_ago_in_words_from_str(date_str, granularity='month'):
-    '''Deprecated in 2.2 use time_ago_from_timestamp'''
-    if date_str:
-        try:
-            return formatters.localised_nice_date(
-                date_str_to_datetime(date_str), show_date=False
-            )
-        except ValueError:
-            pass
-    return _('Unknown')
-
-
 @core_helper
 def time_ago_from_timestamp(timestamp):
     ''' Returns a string like `5 months ago` for a datetime relative to now

ckan/lib/i18n.py

@@ -49,15 +49,14 @@
                         UnknownLocaleError)
 from babel.support import Translations
 import polib
-import six
 
 from ckan.common import config, is_flask_request, aslist
 import ckan.i18n
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import ITranslation
 
 if six.PY2:
-    from pylons import i18n
+    from pylons import i18n as pylons_i18n
     import pylons
 
 
@@ -228,9 +227,10 @@ def _set_lang(lang):
     if config.get('ckan.i18n_directory'):
         fake_config = {'pylons.paths': {'root': config['ckan.i18n_directory']},
                        'pylons.package': config['pylons.package']}
-        i18n.set_lang(lang, pylons_config=fake_config, class_=Translations)
+        pylons_i18n.set_lang(
+            lang, pylons_config=fake_config, class_=Translations)
     else:
-        i18n.set_lang(lang, class_=Translations)
+        pylons_i18n.set_lang(lang, class_=Translations)
 
 
 def handle_request(request, tmpl_context):
@@ -279,10 +279,6 @@ def get_lang():
     if is_flask_request():
         from ckan.config.middleware.flask_app import get_locale
         return get_locale()
-    else:
-        langs = i18n.get_lang()
-        if langs:
-            return langs[0]
     return 'en'
 
 

ckan/lib/navl/dictization_functions.py

@@ -176,8 +176,10 @@ def augment_data(data, schema):
 
     new_data = copy.copy(data)
 
+    keys_to_remove = []
+    junk = {}
+    extras_keys = {}
     # fill junk and extras
-
     for key, value in new_data.items():
         if key in full_schema:
             continue
@@ -190,16 +192,21 @@ def augment_data(data, schema):
                 raise DataError('Only lists of dicts can be placed against '
                                 'subschema %s, not %s' %
                                 (key, type(data[key])))
-
         if key[:-1] in key_combinations:
             extras_key = key[:-1] + ('__extras',)
-            extras = new_data.get(extras_key, {})
+            extras = extras_keys.get(extras_key, {})
             extras[key[-1]] = value
-            new_data[extras_key] = extras
+            extras_keys[extras_key] = extras
         else:
-            junk = new_data.get(("__junk",), {})
             junk[key] = value
-            new_data[("__junk",)] = junk
+        keys_to_remove.append(key)
+
+    if junk:
+        new_data[("__junk",)] = junk
+    for extra_key in extras_keys:
+        new_data[extra_key] = extras_keys[extra_key]
+
+    for key in keys_to_remove:
         new_data.pop(key)
 
     # add missing

ckan/lib/pagination.py

@@ -176,7 +176,8 @@ def __init__(
         # Compute the number of the first and last available page
         if self.item_count > 0:
             self.first_page = 1
-            self.page_count = ((self.item_count - 1) / self.items_per_page) + 1
+            self.page_count = int(
+                ((self.item_count - 1) / self.items_per_page) + 1)
             self.last_page = self.first_page + self.page_count - 1
 
             # Make sure that the requested page number is the range of

ckan/lib/repoze_plugins/friendly_form.py

@@ -30,9 +30,9 @@
 
 from webob import Request
 try:
-    from webob import UnicodeMultiDict as multidict
+    from webob.multidict import MultiDict
 except ImportError:
-    from webob import multidict
+    from webob import UnicodeMultiDict as MultiDict
 
 from webob.exc import HTTPFound, HTTPUnauthorized
 from zope.interface import implementer
@@ -187,7 +187,7 @@ def identify(self, environ):
             #    We are on the URL where repoze.who logs the user out.    #
             r = Request(environ)
             params = dict(list(r.GET.items()) + list(r.POST.items()))
-            form = multidict(params)
+            form = MultiDict(params)
             form.update(query)
             referer = environ.get(u'HTTP_REFERER', script_name)
             came_from = form.get(u'came_from', referer)

ckan/model/__init__.py

@@ -3,13 +3,10 @@
 import warnings
 import logging
 import re
-from datetime import datetime
 from time import sleep
 from os.path import splitext
 
-from six import text_type
 from sqlalchemy import MetaData, __version__ as sqav, Table
-from sqlalchemy.util import OrderedDict
 from sqlalchemy.exc import ProgrammingError
 
 from alembic.command import (

ckan/model/domain_object.py

@@ -1,10 +1,10 @@
 # encoding: utf-8
 
 import datetime
+from collections import OrderedDict
 
 import sqlalchemy as sa
 from sqlalchemy import orm
-from sqlalchemy.util import OrderedDict
 
 from ckan.model import meta, core
 

ckan/model/resource.py

@@ -3,7 +3,7 @@
 import datetime
 
 from six import text_type
-from sqlalchemy.util import OrderedDict
+from collections import OrderedDict
 from sqlalchemy.ext.orderinglist import ordering_list
 from sqlalchemy import orm
 from ckan.common import config

ckan/plugins/toolkit.py

@@ -507,7 +507,9 @@ def _get_endpoint(cls):
                 return common.c.controller, common.c.action
             except AttributeError:
                 return (None, None)
-
+        # service routes, like `static`
+        if len(endpoint) == 1:
+            return endpoint + ('index', )
         return endpoint
 
     def __getattr__(self, name):