Merge pull request #5138 from DataShades/py3-run-tests-2

Green tests using Py3 - Part 2

.circleci/config.yml

@@ -143,4 +143,4 @@ workflows:
   build_and_test:
     jobs:
       - test-python-2
-      # - test-python-3
+      - test-python-3

ckan/cli/cli.py

@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 import logging
+import six
 from collections import defaultdict
 
 import ckan.plugins as p
@@ -73,6 +74,11 @@ def __init__(self, conf=None):
 @click.pass_context
 def ckan(ctx, config, *args, **kwargs):
     ctx.obj = CkanCommand(config)
+    if six.PY2:
+        ctx.meta["flask_app"] = ctx.obj.app.apps["flask_app"]._wsgi_app
+    else:
+        ctx.meta["flask_app"] = ctx.obj.app._wsgi_app
+
     for plugin in p.PluginImplementations(p.IClick):
         for cmd in plugin.get_commands():
             cmd._ckanext = plugin.name

ckan/cli/datastore.py

@@ -88,7 +88,7 @@ def permissions_sql(maindb, datastoredb, mainuser, writeuser, readuser):
 def dump(ctx, resource_id, output_file, format, offset, limit, bom):
     u'''Dump a datastore resource.
     '''
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         dump_to(
             resource_id,

ckan/cli/search_index.py

@@ -5,7 +5,7 @@
 import click
 import sqlalchemy as sa
 
-from ckan.cli import error_shout
+import ckan.plugins.toolkit as tk
 
 
 @click.group(name=u'search-index', short_help=u'Search index commands')
@@ -28,20 +28,17 @@ def search_index():
                    u'ensures that changes are immediately available on the'
                    u'search, but slows significantly the process. Default'
                    u'is false.')
-@click.pass_context
-def rebuild(ctx, verbose, force, refresh, only_missing, quiet, commit_each):
+def rebuild(verbose, force, refresh, only_missing, quiet, commit_each):
     u''' Rebuild search index '''
-    flask_app = ctx.obj.app.apps['flask_app']._wsgi_app
     from ckan.lib.search import rebuild, commit
     try:
-        with flask_app.test_request_context():
-            rebuild(only_missing=only_missing,
-                    force=force,
-                    refresh=refresh,
-                    defer_commit=(not commit_each),
-                    quiet=quiet)
+        rebuild(only_missing=only_missing,
+                force=force,
+                refresh=refresh,
+                defer_commit=(not commit_each),
+                quiet=quiet)
     except Exception as e:
-        error_shout(e)
+        tk.error_shout(e)
     if not commit_each:
         commit()
 
@@ -74,21 +71,19 @@ def clear(dataset_name):
 
 @search_index.command(name=u'rebuild-fast',
                       short_help=u'Reindex with multiprocessing')
-@click.pass_context
-def rebuild_fast(ctx):
-    conf = ctx.obj.config
-    flask_app = ctx.obj.app.apps['flask_app']._wsgi_app
-    db_url = conf['sqlalchemy.url']
+def rebuild_fast():
+    from ckan.lib.search import commit
+
+    db_url = tk.config['sqlalchemy.url']
     engine = sa.create_engine(db_url)
     package_ids = []
     result = engine.execute(u"select id from package where state = 'active';")
     for row in result:
         package_ids.append(row[0])
 
     def start(ids):
-        from ckan.lib.search import rebuild, commit
+        from ckan.lib.search import rebuild
         rebuild(package_ids=ids)
-        commit()
 
     def chunks(l, n):
         u""" Yield n successive chunks from l."""
@@ -98,15 +93,16 @@ def chunks(l, n):
         yield l[n * newn - newn:]
 
     processes = []
-    with flask_app.test_request_context():
-        try:
-            for chunk in chunks(package_ids, mp.cpu_count()):
-                process = mp.Process(target=start, args=(chunk,))
-                processes.append(process)
-                process.daemon = True
-                process.start()
-
-            for process in processes:
-                process.join()
-        except Exception as e:
-            click.echo(e.message)
+
+    try:
+        for chunk in chunks(package_ids, mp.cpu_count()):
+            process = mp.Process(target=start, args=(chunk,))
+            processes.append(process)
+            process.daemon = True
+            process.start()
+
+        for process in processes:
+            process.join()
+        commit()
+    except Exception as e:
+        click.echo(e.message)

ckan/cli/seed.py

@@ -21,39 +21,39 @@ def seed():
 @seed.command(short_help=u'Annakarenina and warandpeace.')
 @click.pass_context
 def basic(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_basic_test_data()
 
 
 @seed.command(short_help=u'Realistic data to test search.')
 @click.pass_context
 def search(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_search_test_data()
 
 
 @seed.command(short_help=u'Government style data.')
 @click.pass_context
 def gov(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_gov_test_data()
 
 
 @seed.command(short_help=u'Package relationships data.')
 @click.pass_context
 def family(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_family_test_data()
 
 
 @seed.command(short_help=u'Create a user "tester" with api key "tester".')
 @click.pass_context
 def user(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_test_user()
     click.echo(
@@ -64,22 +64,22 @@ def user(ctx):
 @seed.command(short_help=u'Test translations of terms.')
 @click.pass_context
 def translations(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_translations_test_data()
 
 
 @seed.command(short_help=u'Some test vocabularies.')
 @click.pass_context
 def vocabs(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_vocabs_test_data()
 
 
 @seed.command(short_help=u'Hierarchy of groups.')
 @click.pass_context
 def hierarchy(ctx):
-    flask_app = ctx.obj.app.apps[u'flask_app']._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         CreateTestData.create_group_hierarchy_test_data()

ckan/cli/sysadmin.py

@@ -43,16 +43,15 @@ def list_sysadmins():
 @sysadmin.command(help=u"Convert user into a sysadmin.")
 @click.argument(u"username")
 @click.argument(u"args", nargs=-1)
-def add(username, args):
-
+@click.pass_context
+def add(ctx, username, args):
     user = model.User.by_name(text_type(username))
     if not user:
         click.secho(u'User "%s" not found' % username, fg=u"red")
         if click.confirm(
             u"Create new user: %s?" % username, default=True, abort=True
         ):
-            # TODO: once, cli.user is merged, invoke `user.add_user` instead
-            add_user([username] + list(args))
+            ctx.forward(add_user)
             user = model.User.by_name(text_type(username))
 
     user.sysadmin = True

ckan/cli/user.py

@@ -4,6 +4,7 @@
 import sys
 from pprint import pprint
 
+import six
 import click
 from six import text_type
 
@@ -24,7 +25,7 @@ def user():
 @click.argument(u'username')
 @click.argument(u'args', nargs=-1)
 @click.pass_context
-def add_user(username, args):
+def add_user(ctx, username, args):
     u'''Add new user if we use ckan sysadmin add
     or ckan user add
     '''
@@ -69,11 +70,16 @@ def add_user(username, args):
             u'ignore_auth': True,
             u'user': site_user['name'],
         }
-        user_dict = logic.get_action(u'user_create')(context, data_dict)
+        flask_app = ctx.meta['flask_app']
+        # Current user is tested agains sysadmin role during model
+        # dictization, thus we need request context
+        with flask_app.test_request_context():
+            user_dict = logic.get_action(u'user_create')(context, data_dict)
         click.secho(u"Successfully created user: %s" % user_dict['name'],
                     fg=u'green', bold=True)
     except logic.ValidationError as e:
         error_shout(e)
+        raise click.Abort()
 
 
 def get_user_str(user):
@@ -102,10 +108,9 @@ def remove_user(ctx, username):
         error_shout(u'Please specify the username to be removed')
         return
 
-    flask_app = ctx.obj.app.apps['flask_app']._wsgi_app
     site_user = logic.get_action(u'get_site_user')({u'ignore_auth': True}, {})
     context = {u'user': site_user[u'name']}
-    with flask_app.test_request_context():
+    with ctx.meta['flask_app'].test_request_context():
         plugin.toolkit.get_action(u'user_delete')(context, {u'id': username})
         click.secho(u'Deleted user: %s' % username, fg=u'green', bold=True)
 

ckan/cli/views.py

@@ -47,7 +47,7 @@ def create(ctx, types, dataset, no_default_filters, search, yes):
         u"datastore" in p.toolkit.config[u"ckan.plugins"].split()
     )
 
-    flask_app = ctx.obj.app.apps[u"flask_app"]._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         loaded_view_plugins = _get_view_plugins(types, datastore_enabled)
     if loaded_view_plugins is None:
@@ -148,7 +148,7 @@ def clean(ctx, yes):
 
     """
     names = []
-    flask_app = ctx.obj.app.apps[u"flask_app"]._wsgi_app
+    flask_app = ctx.meta['flask_app']
     with flask_app.test_request_context():
         for plugin in p.PluginImplementations(p.IResourceView):
             names.append(str(plugin.info()[u"name"]))

ckan/config/middleware/common_middleware.py

@@ -70,12 +70,12 @@ def __call__(self, environ, start_response):
         if path == '/_tracking' and method == 'POST':
             # do the tracking
             # get the post data
-            payload = environ['wsgi.input'].read()
+            payload = six.ensure_str(environ['wsgi.input'].read())
             parts = payload.split('&')
             data = {}
             for part in parts:
                 k, v = part.split('=')
-                data[k] = six.ensure_text(unquote(v))
+                data[k] = unquote(v)
             start_response('200 OK', [('Content-Type', 'text/html')])
             # we want a unique anonomized key for each user so that we do
             # not count multiple clicks from the same user.
@@ -85,7 +85,7 @@ def __call__(self, environ, start_response):
                 environ.get('HTTP_ACCEPT_LANGUAGE', ''),
                 environ.get('HTTP_ACCEPT_ENCODING', ''),
             ])
-            key = hashlib.md5(key).hexdigest()
+            key = hashlib.md5(six.ensure_binary(key)).hexdigest()
             # store key/data here
             sql = '''INSERT INTO tracking_raw
                      (user_key, url, tracking_type)

ckan/config/middleware/flask_app.py

@@ -30,6 +30,7 @@
 from ckan.lib import helpers
 from ckan.lib import jinja_extensions
 from ckan.common import config, g, request, ungettext
+from ckan.config.middleware.common_middleware import TrackingMiddleware
 import ckan.lib.app_globals as app_globals
 import ckan.lib.plugins as lib_plugins
 import ckan.plugins.toolkit as toolkit
@@ -300,6 +301,9 @@ def ungettext_alias():
     if six.PY3:
         app = I18nMiddleware(app)
 
+        if asbool(config.get('ckan.tracking_enabled', 'false')):
+            app = TrackingMiddleware(app, config)
+
     # Add a reference to the actual Flask app so it's easier to access
     app._wsgi_app = flask_app
 

ckan/lib/dictization/__init__.py

@@ -24,6 +24,7 @@
 # and saving dictized objects. If a specialised use is needed please do NOT extend
 # these functions.  Copy code from here as needed.
 
+legacy_dict_sort = lambda x: (len(x), dict.items(x))
 
 def table_dictize(obj, context, **kw):
     '''Get any model object and represent it as a dict'''
@@ -72,7 +73,7 @@ def table_dictize(obj, context, **kw):
     return result_dict
 
 
-def obj_list_dictize(obj_list, context, sort_key=lambda x:x):
+def obj_list_dictize(obj_list, context, sort_key=legacy_dict_sort):
     '''Get a list of model object and represent it as a list of dicts'''
 
     result_list = []

ckan/lib/hash.py

@@ -2,6 +2,7 @@
 
 import hmac
 import hashlib
+import six
 
 from ckan.common import config, request
 
@@ -12,7 +13,7 @@ def get_message_hash(value):
     if not secret:
         # avoid getting config value at module scope since config may
         # not be read in yet
-        secret = config['beaker.session.secret']
+        secret = six.ensure_binary(config['beaker.session.secret'])
     return hmac.new(secret, value.encode('utf8'), hashlib.sha1).hexdigest()
 
 def get_redirect():

ckan/lib/helpers.py

@@ -1666,8 +1666,7 @@ def date_str_to_datetime(date_str):
 
 @core_helper
 def parse_rfc_2822_date(date_str, assume_utc=True):
-    '''
-    Parse a date string of the form specified in RFC 2822, and return a
+    '''Parse a date string of the form specified in RFC 2822, and return a
     datetime.
 
     RFC 2822 is the date format used in HTTP headers.  It should contain
@@ -1684,6 +1683,10 @@ def parse_rfc_2822_date(date_str, assume_utc=True):
     datetime is 'aware', ie - it has an associated tz_info object.
 
     Returns None if the string cannot be parsed as a valid datetime.
+
+    Note: in Python3, `email.utils` always assume UTC if there is no
+    timezone, so `assume_utc` has no sense in this version.
+
     '''
     time_tuple = email.utils.parsedate_tz(date_str)
 

ckan/lib/navl/validators.py

@@ -65,7 +65,7 @@ def empty(key, data, errors, context):
         key_name = key[-1]
         if key_name == '__junk':
             # for junked fields, the field name is contained in the value
-            key_name = value.keys()
+            key_name = list(value.keys())
         errors[key].append(_(
             'The input field %(name)s was not expected.') % {"name": key_name})
 

ckan/lib/search/__init__.py

@@ -158,7 +158,7 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False,
         log.info('Indexing just package %r...', pkg_dict['name'])
         package_index.remove_dict(pkg_dict)
         package_index.insert_dict(pkg_dict)
-    elif package_ids:
+    elif package_ids is not None:
         for package_id in package_ids:
             pkg_dict = logic.get_action('package_show')(context,
                 {'id': package_id})