From 6b6a6c2d248f44662f76f2bd443bbb1944295741 Mon Sep 17 00:00:00 2001
From: howff <3064316+howff@users.noreply.github.com>
Date: Thu, 14 Nov 2019 12:36:49 +0000
Subject: [PATCH 01/30] Change to root_path also needs who.ini edited

---
 doc/maintaining/configuration.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst
index d95b55c7f5d..12823a0d202 100644
--- a/doc/maintaining/configuration.rst
+++ b/doc/maintaining/configuration.rst
@@ -1925,6 +1925,10 @@ This setting is used to construct URLs inside CKAN. It specifies two things:
 
     The host of your CKAN installation can be set via :ref:`ckan.site_url`.
 
+The CKAN repoze config file ``who.ini`` file will also need to be edited
+by adding the path prefix to the options in the ``[plugin:friendlyform]``
+section: ``login_form_url``, ``post_login_url`` and ``post_logout_url``.
+Do not change the login/logout_handler_path options.
 
 .. _ckan.resource_formats:
 

From 5d8dcdc6aaee79674407b4941d5dea2de4988c0d Mon Sep 17 00:00:00 2001
From: howff <3064316+howff@users.noreply.github.com>
Date: Tue, 19 Nov 2019 11:19:53 +0000
Subject: [PATCH 02/30] parseCompletions handles v3 api result

v2 api returns a ResultSet: { Result: { Name: xxx } } but the v3 api returns result: [ xxx ] so map the latter to the former.
---
 ckan/public/base/javascript/client.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ckan/public/base/javascript/client.js b/ckan/public/base/javascript/client.js
index 9dd75ea522f..b7fe576ebd0 100644
--- a/ckan/public/base/javascript/client.js
+++ b/ckan/public/base/javascript/client.js
@@ -160,6 +160,9 @@
       }
 
       var map = {};
+      // If given a 'result' array then convert it into a Result dict inside a Result dict.
+      data = data.result ? { 'ResultSet': { 'Result': data.result.map(x => ({'Name': x})) } } : data;
+      // If given a Result dict inside a ResultSet dict then use the Result dict.
       var raw = jQuery.isArray(data) ? data : data.ResultSet && data.ResultSet.Result || {};
 
       var items = jQuery.map(raw, function (item) {

From 032ee106cc4ae51880087243bd25bfa3b9a6ab59 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 21 Nov 2019 15:35:30 +0100
Subject: [PATCH 03/30] [#4801] string.letters not present in py3

---
 ckan/lib/search/index.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py
index 282f32d565c..548f33a8b22 100644
--- a/ckan/lib/search/index.py
+++ b/ckan/lib/search/index.py
@@ -10,6 +10,7 @@
 
 import re
 
+import six
 import pysolr
 from ckan.common import config
 from ckan.common import asbool
@@ -29,7 +30,11 @@
 
 TYPE_FIELD = "entity_type"
 PACKAGE_TYPE = "package"
-KEY_CHARS = string.digits + string.letters + "_-"
+if six.PY2:
+    KEY_CHARS = string.digits + string.letters + "_-"
+else:
+    KEY_CHARS = string.digits + string.ascii_letters + "_-"
+
 SOLR_FIELDS = [TYPE_FIELD, "res_url", "text", "urls", "indexed_ts", "site_id"]
 RESERVED_FIELDS = SOLR_FIELDS + ["tags", "groups", "res_name", "res_description",
                                  "res_format", "res_url", "res_type"]

From 3e2efb621c0f84f7a0fb73f849993c5839aaea24 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 21 Nov 2019 15:42:19 +0100
Subject: [PATCH 04/30] [#4800] Remove formencode support

---
 ckan/config/environment.py              |  7 -----
 ckan/lib/navl/dictization_functions.py  | 18 -------------
 ckan/tests/legacy/lib/test_navl.py      | 35 -------------------------
 ckan/tests/lib/test_navl.py             | 32 ----------------------
 doc/extensions/adding-custom-fields.rst |  9 +++----
 5 files changed, 3 insertions(+), 98 deletions(-)
 delete mode 100644 ckan/tests/lib/test_navl.py

diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index f96c3e7c0bd..3696664b0b2 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -8,7 +8,6 @@
 
 import sqlalchemy
 from pylons import config as pylons_config
-import formencode
 
 from six.moves.urllib.parse import urlparse
 
@@ -272,12 +271,6 @@ def update_config():
         template_paths = extra_template_paths.split(',') + template_paths
     config['computed_template_paths'] = template_paths
 
-    # Set the default language for validation messages from formencode
-    # to what is set as the default locale in the config
-    default_lang = config.get('ckan.locale_default', 'en')
-    formencode.api.set_stdtranslation(domain="FormEncode",
-                                      languages=[default_lang])
-
     # Markdown ignores the logger config, so to get rid of excessive
     # markdown debug messages in the log, set it to the level of the
     # root logger.
diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py
index 8377ad4dfc0..3f2275b17e4 100644
--- a/ckan/lib/navl/dictization_functions.py
+++ b/ckan/lib/navl/dictization_functions.py
@@ -1,8 +1,6 @@
 # encoding: utf-8
 
 import copy
-import formencode as fe
-import inspect
 import json
 
 from six import text_type
@@ -214,22 +212,6 @@ def augment_data(data, schema):
 
 def convert(converter, key, converted_data, errors, context):
 
-    if inspect.isclass(converter) and issubclass(converter, fe.Validator):
-        try:
-            value = converted_data.get(key)
-            value = converter().to_python(value, state=context)
-        except fe.Invalid as e:
-            errors[key].append(e.msg)
-        return
-
-    if isinstance(converter, fe.Validator):
-        try:
-            value = converted_data.get(key)
-            value = converter.to_python(value, state=context)
-        except fe.Invalid as e:
-            errors[key].append(e.msg)
-        return
-
     try:
         value = converter(converted_data.get(key))
         converted_data[key] = value
diff --git a/ckan/tests/legacy/lib/test_navl.py b/ckan/tests/legacy/lib/test_navl.py
index c884c60fd91..f8e11222b57 100644
--- a/ckan/tests/legacy/lib/test_navl.py
+++ b/ckan/tests/legacy/lib/test_navl.py
@@ -20,8 +20,6 @@
                         convert_int,
                         ignore)
 
-from formencode import validators
-
 
 schema = {
     "__after": [identity_converter],
@@ -306,39 +304,6 @@ def test_simple_converter_types():
     assert isinstance(converted_data["gender"], str)
 
 
-def test_formencode_compat():
-    schema = {
-        "name": [not_empty, text_type],
-        "email": [validators.Email],
-        "email2": [validators.Email],
-    }
-
-    data = {
-        "name": "fred",
-        "email": "32",
-        "email2": "david@david.com",
-    }
-
-    converted_data, errors = validate(data, schema)
-    assert errors == {'email': [u'An email address must contain a single @']}, errors
-
-def test_range_validator():
-
-    schema = {
-        "name": [not_empty, text_type],
-        "email": [validators.Int(min=1, max=10)],
-        "email2": [validators.Email],
-    }
-
-    data = {
-        "email": "32",
-        "email2": "david@david.com",
-    }
-
-    converted_data, errors = validate(data, schema)
-    assert errors == {'name': [u'Missing value'], 'email': [u'Please enter a number that is 10 or smaller']}, errors
-
-
 def validate_flattened(data, schema, context=None):
 
     context = context or {}
diff --git a/ckan/tests/lib/test_navl.py b/ckan/tests/lib/test_navl.py
deleted file mode 100644
index 009a0c5c87d..00000000000
--- a/ckan/tests/lib/test_navl.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# encoding: utf-8
-
-import nose
-from six import text_type
-
-from ckan.tests import helpers
-from ckan.config import environment
-
-eq_ = nose.tools.eq_
-
-
-class TestFormencdoeLanguage(object):
-    @helpers.change_config('ckan.locale_default', 'de')
-    def test_formencode_uses_locale_default(self):
-        environment.update_config()
-        from ckan.lib.navl.dictization_functions import validate
-        from ckan.lib.navl.validators import not_empty
-        from formencode import validators
-        schema = {
-            "name": [not_empty, text_type],
-            "email": [validators.Email],
-            "email2": [validators.Email],
-        }
-
-        data = {
-            "name": "fred",
-            "email": "32",
-            "email2": "david@david.com",
-        }
-
-        converted_data, errors = validate(data, schema)
-        eq_({'email': [u'Eine E-Mail-Adresse muss genau ein @-Zeichen enthalten']}, errors)
diff --git a/doc/extensions/adding-custom-fields.rst b/doc/extensions/adding-custom-fields.rst
index 04f22a49f90..a1fd0ea2269 100644
--- a/doc/extensions/adding-custom-fields.rst
+++ b/doc/extensions/adding-custom-fields.rst
@@ -213,16 +213,13 @@ Any of the following objects may be used as validators as part
 of a custom dataset, group or organization schema. CKAN's validation
 code will check for and attempt to use them in this order:
 
-1. a `formencode Validator class <http://www.formencode.org/en/latest/Validator.html>`_ (not discussed)
 
-2. a formencode Validator instance (not discussed)
+1. a callable object taking a single parameter: ``validator(value)``
 
-3. a callable object taking a single parameter: ``validator(value)``
-
-4. a callable object taking four parameters:
+2. a callable object taking four parameters:
    ``validator(key, flattened_data, errors, context)``
 
-5. a callable object taking two parameters
+3. a callable object taking two parameters
    ``validator(value, context)``
 
 

From 5db6a46938b4a65bd817f44d268319291a6fca42 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 21 Nov 2019 18:12:51 +0100
Subject: [PATCH 05/30] [#4801] Conditional imports for pylons / py2 only stuff

The webhelpers ones should be removed once #4794 is done
---
 ckan/config/environment.py                    |  21 +-
 ckan/lib/base.py                              |  84 +++--
 ckan/lib/helpers.py                           |  97 +++---
 ckan/lib/i18n.py                              |  11 +-
 .../migration/revision_legacy_code_tests.py   | 307 ++++++++++++++++++
 5 files changed, 420 insertions(+), 100 deletions(-)
 create mode 100644 ckan/tests/migration/revision_legacy_code_tests.py

diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index 3696664b0b2..678226febdb 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -6,8 +6,8 @@
 import warnings
 import pytz
 
+import six
 import sqlalchemy
-from pylons import config as pylons_config
 
 from six.moves.urllib.parse import urlparse
 
@@ -20,7 +20,6 @@
 from ckan.lib.redis import is_redis_available
 import ckan.lib.render as render
 import ckan.lib.search as search
-import ckan.lib.plugins as lib_plugins
 import ckan.logic as logic
 import ckan.authz as authz
 import ckan.lib.jinja_extensions as jinja_extensions
@@ -30,6 +29,10 @@
 from ckan.common import _, ungettext, config
 from ckan.exceptions import CkanConfigurationException
 
+if six.PY2:
+    from pylons import config as pylons_config
+
+
 log = logging.getLogger(__name__)
 
 
@@ -97,13 +100,15 @@ def find_controller(self, controller):
     config.update(global_conf)
     config.update(app_conf)
 
-    # Initialize Pylons own config object
-    pylons_config.init_app(global_conf, app_conf, package='ckan', paths=paths)
+    if six.PY2:
+        # Initialize Pylons own config object
+        pylons_config.init_app(
+            global_conf, app_conf, package='ckan', paths=paths)
 
-    # Update the main CKAN config object with the Pylons specific stuff, as it
-    # is quite hard to keep them separated. This should be removed once Pylons
-    # support is dropped
-    config.update(pylons_config)
+        # Update the main CKAN config object with the Pylons specific stuff,
+        # as it is quite hard to keep them separated. This should be removed
+        # once Pylons support is dropped
+        config.update(pylons_config)
 
     # Setup the SQLAlchemy database engine
     # Suppress a couple of sqlalchemy warnings
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index 4201ae44d35..d73b98446c3 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -9,38 +9,36 @@
 import inspect
 import sys
 
-from pylons import cache
-from pylons.controllers import WSGIController
-from pylons.controllers.util import abort as _abort
-from pylons.decorators import jsonify
-from pylons.templating import cached_template, pylons_globals
-from webhelpers.html import literal
 from jinja2.exceptions import TemplateNotFound
 
 from flask import (
     render_template as flask_render_template,
     abort as flask_abort
 )
-import ckan.exceptions
-import ckan
+import six
+
 import ckan.lib.i18n as i18n
 import ckan.lib.render as render_
 import ckan.lib.helpers as h
 import ckan.lib.app_globals as app_globals
 import ckan.plugins as p
 import ckan.model as model
-
 from ckan.views import (identify_user,
                         set_cors_headers_for_response,
                         check_session_cookie,
                         )
-
-# These imports are for legacy usages and will be removed soon these should
-# be imported directly from ckan.common for internal ckan code and via the
-# plugins.toolkit for extensions.
-from ckan.common import (json, _, ungettext, c, request, response, config,
+from ckan.common import (c, request, config,
                          session, is_flask_request)
 
+
+if six.PY2:
+    from pylons.controllers import WSGIController
+    from pylons.controllers.util import abort as _abort
+    from pylons.templating import cached_template, pylons_globals
+    from webhelpers.html import literal
+    from ckan.common import response
+
+
 log = logging.getLogger(__name__)
 
 APIKEY_HEADER_NAME_KEY = 'apikey_header_name'
@@ -238,45 +236,41 @@ class ValidationException(Exception):
     pass
 
 
-class BaseController(WSGIController):
-    '''Base class for CKAN controller classes to inherit from.
+if six.PY2:
+    class BaseController(WSGIController):
+        '''Base class for CKAN controller classes to inherit from.
 
-    '''
-    repo = model.repo
-    log = logging.getLogger(__name__)
-
-    def __before__(self, action, **params):
-        c.__timer = time.time()
-        app_globals.app_globals._check_uptodate()
+        '''
+        repo = model.repo
+        log = logging.getLogger(__name__)
 
-        identify_user()
+        def __before__(self, action, **params):
+            c.__timer = time.time()
+            app_globals.app_globals._check_uptodate()
 
-        i18n.handle_request(request, c)
+            identify_user()
 
-    def __call__(self, environ, start_response):
-        """Invoke the Controller"""
-        # WSGIController.__call__ dispatches to the Controller method
-        # the request is routed to. This routing information is
-        # available in environ['pylons.routes_dict']
+            i18n.handle_request(request, c)
 
-        try:
-            res = WSGIController.__call__(self, environ, start_response)
-        finally:
-            model.Session.remove()
+        def __call__(self, environ, start_response):
+            """Invoke the Controller"""
+            # WSGIController.__call__ dispatches to the Controller method
+            # the request is routed to. This routing information is
+            # available in environ['pylons.routes_dict']
 
-        check_session_cookie(response)
-
-        return res
+            try:
+                res = WSGIController.__call__(self, environ, start_response)
+            finally:
+                model.Session.remove()
 
-    def __after__(self, action, **params):
+            check_session_cookie(response)
 
-        set_cors_headers_for_response(response)
+            return res
 
-        r_time = time.time() - c.__timer
-        url = request.environ['CKAN_CURRENT_URL'].split('?')[0]
-        log.info(' %s render time %.3f seconds' % (url, r_time))
+        def __after__(self, action, **params):
 
+            set_cors_headers_for_response(response)
 
-# Include the '_' function in the public names
-__all__ = [__name for __name in locals().keys() if not __name.startswith('_')
-           or __name == '_']
+            r_time = time.time() - c.__timer
+            url = request.environ['CKAN_CURRENT_URL'].split('?')[0]
+            log.info(' %s render time %.3f seconds' % (url, r_time))
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index 7c68198ba1d..3335ef9d76c 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -17,22 +17,17 @@
 import uuid
 
 from paste.deploy import converters
-from webhelpers.html import HTML, literal, tags
-from webhelpers import paginate
-import webhelpers.text as whtext
-import webhelpers.date as date
+
 from markdown import markdown
 from bleach import clean as bleach_clean, ALLOWED_TAGS, ALLOWED_ATTRIBUTES
-from pylons import url as _pylons_default_url
 from ckan.common import config, is_flask_request
 from flask import redirect as _flask_redirect
 from flask import _request_ctx_stack
-from routes import redirect_to as _routes_redirect_to
-from routes import url_for as _routes_default_url_for
 from flask import url_for as _flask_default_url_for
 from werkzeug.routing import BuildError as FlaskRouteBuildError
 from ckan.lib import i18n
 
+import six
 from six import string_types, text_type
 from six.moves.urllib.parse import (
     urlencode, quote, unquote, urlparse, urlunparse
@@ -55,6 +50,18 @@
 from ckan.lib.webassets_tools import include_asset, render_assets
 from markupsafe import Markup, escape
 
+if six.PY2:
+    # TODO: webhelpers should be removed after #4794 is done
+    from webhelpers.html import HTML, literal, tags
+    from webhelpers import paginate
+    import webhelpers.text as whtext
+    import webhelpers.date as date
+
+    from pylons import url as _pylons_default_url
+    from routes import redirect_to as _routes_redirect_to
+    from routes import url_for as _routes_default_url_for
+
+
 
 log = logging.getLogger(__name__)
 
@@ -1358,43 +1365,45 @@ def pager_url(page, partial=None, **kwargs):
     return url_for(*pargs, **kwargs)
 
 
-class Page(paginate.Page):
-    # Curry the pager method of the webhelpers.paginate.Page class, so we have
-    # our custom layout set as default.
+# TODO: remove once #4794 is done
+if six.PY2:
+    class Page(paginate.Page):
+        # Curry the pager method of the webhelpers.paginate.Page class, so we have
+        # our custom layout set as default.
 
-    def pager(self, *args, **kwargs):
-        kwargs.update(
-            format=u"<div class='pagination-wrapper'><ul class='pagination'>"
-            "$link_previous ~2~ $link_next</ul></div>",
-            symbol_previous=u'«', symbol_next=u'»',
-            curpage_attr={'class': 'active'}, link_attr={}
-        )
-        return super(Page, self).pager(*args, **kwargs)
+        def pager(self, *args, **kwargs):
+            kwargs.update(
+                format=u"<div class='pagination-wrapper'><ul class='pagination'>"
+                "$link_previous ~2~ $link_next</ul></div>",
+                symbol_previous=u'«', symbol_next=u'»',
+                curpage_attr={'class': 'active'}, link_attr={}
+            )
+            return super(Page, self).pager(*args, **kwargs)
 
-    # Put each page link into a <li> (for Bootstrap to style it)
+        # Put each page link into a <li> (for Bootstrap to style it)
 
-    def _pagerlink(self, page, text, extra_attributes=None):
-        anchor = super(Page, self)._pagerlink(page, text)
-        extra_attributes = extra_attributes or {}
-        return HTML.li(anchor, **extra_attributes)
+        def _pagerlink(self, page, text, extra_attributes=None):
+            anchor = super(Page, self)._pagerlink(page, text)
+            extra_attributes = extra_attributes or {}
+            return HTML.li(anchor, **extra_attributes)
 
-    # Change 'current page' link from <span> to <li><a>
-    # and '..' into '<li><a>..'
-    # (for Bootstrap to style them properly)
+        # Change 'current page' link from <span> to <li><a>
+        # and '..' into '<li><a>..'
+        # (for Bootstrap to style them properly)
 
-    def _range(self, regexp_match):
-        html = super(Page, self)._range(regexp_match)
-        # Convert ..
-        dotdot = '<span class="pager_dotdot">..</span>'
-        dotdot_link = HTML.li(HTML.a('...', href='#'), class_='disabled')
-        html = re.sub(dotdot, dotdot_link, html)
+        def _range(self, regexp_match):
+            html = super(Page, self)._range(regexp_match)
+            # Convert ..
+            dotdot = '<span class="pager_dotdot">..</span>'
+            dotdot_link = HTML.li(HTML.a('...', href='#'), class_='disabled')
+            html = re.sub(dotdot, dotdot_link, html)
 
-        # Convert current page
-        text = '%s' % self.page
-        current_page_span = str(HTML.span(c=text, **self.curpage_attr))
-        current_page_link = self._pagerlink(self.page, text,
-                                            extra_attributes=self.curpage_attr)
-        return re.sub(current_page_span, current_page_link, html)
+            # Convert current page
+            text = '%s' % self.page
+            current_page_span = str(HTML.span(c=text, **self.curpage_attr))
+            current_page_link = self._pagerlink(self.page, text,
+                                                extra_attributes=self.curpage_attr)
+            return re.sub(current_page_span, current_page_link, html)
 
 
 @core_helper
@@ -2640,11 +2649,13 @@ def clean_html(html):
 core_helper(i18n.get_available_locales)
 core_helper(i18n.get_locales_dict)
 # Useful additions from the webhelpers library.
-core_helper(tags.literal)
-core_helper(tags.link_to)
-core_helper(tags.file)
-core_helper(tags.submit)
-core_helper(whtext.truncate)
+# TODO: remove once #4794 is done
+if six.PY2:
+    core_helper(tags.literal)
+    core_helper(tags.link_to)
+    core_helper(tags.file)
+    core_helper(tags.submit)
+    core_helper(whtext.truncate)
 # Useful additions from the paste library.
 core_helper(converters.asbool)
 # Useful additions from the stdlib.
diff --git a/ckan/lib/i18n.py b/ckan/lib/i18n.py
index 1844c3ce6a6..1655facf9b2 100644
--- a/ckan/lib/i18n.py
+++ b/ckan/lib/i18n.py
@@ -42,21 +42,24 @@
 import os
 import os.path
 
+import six
 from babel import Locale
 from babel.core import (LOCALE_ALIASES,
                         get_locale_identifier,
                         UnknownLocaleError)
 from babel.support import Translations
-from ckan.common import aslist
-from pylons import i18n
-import pylons
 import polib
 
-from ckan.common import config, is_flask_request
+from ckan.common import config, is_flask_request, aslist
 import ckan.i18n
 from ckan.plugins import PluginImplementations
 from ckan.plugins.interfaces import ITranslation
 
+if six.PY2:
+    from pylons import i18n
+    import pylons
+
+
 
 log = logging.getLogger(__name__)
 
diff --git a/ckan/tests/migration/revision_legacy_code_tests.py b/ckan/tests/migration/revision_legacy_code_tests.py
new file mode 100644
index 00000000000..780d7c18439
--- /dev/null
+++ b/ckan/tests/migration/revision_legacy_code_tests.py
@@ -0,0 +1,307 @@
+# encoding: utf-8
+
+# This file is at the top of the ckan repository, because it needs to be run
+# separately from all the other tests, because when it imports
+# revision_legacy_code.py it changes the core model, which causes a number of
+# test failures which we're not concerned about.
+
+from difflib import unified_diff
+from pprint import pprint, pformat
+
+from ckan import model
+
+import ckan.lib.search as search
+from ckan.lib.dictization.model_save import package_dict_save
+from ckan.lib.create_test_data import CreateTestData
+from ckan.tests import helpers
+
+from ckan.migration.revision_legacy_code import package_dictize_with_revisions as package_dictize
+from ckan.migration.revision_legacy_code import RevisionTableMappings, make_package_revision
+from ckan.migration.migrate_package_activity import PackageDictizeMonkeyPatch
+
+
+# tests here have been moved from ckan/tests/legacy/lib/test_dictization.py
+class TestPackageDictizeWithRevisions(object):
+    @classmethod
+    def setup_class(cls):
+
+        cls.package_expected = {
+            u'author': None,
+            u'author_email': None,
+            u'creator_user_id': None,
+            'extras': [
+                # extra_revision_table is no longer being populated because
+                # PackageExtra no longer has
+                # vdm.sqlalchemy.Revisioner(extra_revision_table) (removed in
+                # #4691) so don't test extras for the moment
+                # {'key': u'david', 'state': u'active', 'value': u'new_value'},
+                # {'key': u'genre', 'state': u'active', 'value': u'new_value'},
+                # {'key': u'original media', 'state': u'active',
+                #  'value': u'book'}
+                 ],
+            'groups': [{
+                        u'name': u'david',
+                        u'capacity': u'public',
+                        u'image_url': u'',
+                        u'image_display_url': u'',
+                        u'description': u'These are books that David likes.',
+                        u'display_name': u"Dave's books",
+                        u'type': u'group',
+                        u'state': u'active',
+                        u'is_organization': False,
+                        u'title': u"Dave's books",
+                        u"approval_status": u"approved"},
+                       {
+                        u'name': u'roger',
+                        u'capacity': u'public',
+                        u'description': u'Roger likes these books.',
+                        u'image_url': u'',
+                        'image_display_url': u'',
+                        'display_name': u"Roger's books",
+                        u'type': u'group',
+                        u'state': u'active',
+                        u'is_organization': False,
+                        u'title': u"Roger's books",
+                        u"approval_status": u"approved"}],
+            'isopen': True,
+            u'license_id': u'other-open',
+            'license_title': u'Other (Open)',
+            'organization': None,
+            u'owner_org': None,
+            u'maintainer': None,
+            u'maintainer_email': None,
+            u'name': u'annakarenina',
+            u'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
+            'num_resources': 2,
+            'num_tags': 3,
+            u'private': False,
+            'relationships_as_object': [],
+            'relationships_as_subject': [],
+            'resources': [{u'alt_url': u'alt123',
+                            u'cache_last_updated': None,
+                            u'cache_url': None,
+                            u'description': u'Full text. Needs escaping: " Umlaut: \xfc',
+                            u'format': u'plain text',
+                            u'hash': u'abc123',
+                            u'last_modified': None,
+                            u'mimetype': None,
+                            u'mimetype_inner': None,
+                            u'name': None,
+                            u'position': 0,
+                            u'resource_type': None,
+                            u'size': None,
+                            u'size_extra': u'123',
+                            u'url_type': None,
+                            u'state': u'active',
+                            u'url': u'http://datahub.io/download/x=1&y=2',},
+                           {u'alt_url': u'alt345',
+                            u'cache_last_updated': None,
+                            u'cache_url': None,
+                            u'description': u'Index of the novel',
+                            u'format': u'JSON',
+                            u'hash': u'def456',
+                            u'last_modified': None,
+                            u'mimetype': None,
+                            u'mimetype_inner': None,
+                            u'name': None,
+                            u'position': 1,
+                            u'resource_type': None,
+                            u'url_type': None,
+                            u'size': None,
+                            u'size_extra': u'345',
+                            u'state': u'active',
+                            u'url': u'http://datahub.io/index.json'}],
+            u'state': u'active',
+            'tags': [{u'name': u'Flexible \u30a1',
+                        'display_name': u'Flexible \u30a1',
+                        u'state': u'active'},
+                     {'display_name': u'russian',
+                         u'name': u'russian',
+                         u'state': u'active'},
+                     {'display_name': u'tolstoy',
+                         u'name': u'tolstoy',
+                         u'state': u'active'}],
+            u'title': u'A Novel By Tolstoy',
+            u'type': u'dataset',
+            u'url': u'http://datahub.io',
+            u'version': u'0.7a',
+        }
+
+    def setup(self):
+        helpers.reset_db()
+        search.clear_all()
+        CreateTestData.create()
+        make_package_revision(model.Package.by_name('annakarenina'))
+
+    def teardown(self):
+        helpers.reset_db()
+        search.clear_all()
+
+    def test_09_package_alter(self):
+
+        context = {"model": model,
+                   "session": model.Session,
+                   "user": 'testsysadmin'
+                   }
+
+        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one()
+
+        anna_dictized = package_dictize(anna1, context)
+
+        anna_dictized["name"] = u'annakarenina_changed'
+        anna_dictized["resources"][0]["url"] = u'http://new_url'
+
+        package_dict_save(anna_dictized, context)
+        model.Session.commit()
+        model.Session.remove()
+        make_package_revision(model.Package.by_name('annakarenina_changed'))
+
+        pkg = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one()
+
+        package_dictized = package_dictize(pkg, context)
+
+        resources_revisions = model.Session.query(RevisionTableMappings.instance().ResourceRevision).filter_by(package_id=anna1.id).all()
+
+        sorted_resource_revisions = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1]
+        for res in sorted_resource_revisions:
+            print(res.id, res.revision_timestamp, res.state)
+        assert len(sorted_resource_revisions) == 4  # 2 resources originally, then make_package_revision saves them both again
+
+        # Make sure we remove changeable fields BEFORE we store the pretty-printed version
+        # for comparison
+        clean_package_dictized = self.remove_changable_columns(package_dictized)
+
+        anna_original = pformat(anna_dictized)
+        anna_after_save = pformat(clean_package_dictized)
+
+        assert self.remove_changable_columns(anna_dictized) == clean_package_dictized, \
+            "\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
+
+        # changes to the package, relied upon by later tests
+        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one()
+        anna_dictized = package_dictize(anna1, context)
+        anna_dictized['name'] = u'annakarenina_changed2'
+        anna_dictized['resources'][0]['url'] = u'http://new_url2'
+        anna_dictized['tags'][0]['name'] = u'new_tag'
+        anna_dictized['tags'][0].pop('id')  # test if
+        anna_dictized['extras'][0]['value'] = u'new_value'
+
+        package_dict_save(anna_dictized, context)
+        model.Session.commit()
+        model.Session.remove()
+        make_package_revision(model.Package.by_name('annakarenina_changed2'))
+
+        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one()
+        anna_dictized = package_dictize(anna1, context)
+        anna_dictized['notes'] = 'wee'
+        anna_dictized['resources'].append({
+                            'format': u'plain text',
+                            'url': u'http://newurl'}
+                            )
+        anna_dictized['tags'].append({'name': u'newnew_tag'})
+        anna_dictized['extras'].append({'key': 'david',
+                                        'value': u'new_value'})
+
+        package_dict_save(anna_dictized, context)
+        model.Session.commit()
+        model.Session.remove()
+        make_package_revision(model.Package.by_name('annakarenina_changed2'))
+
+        context = {'model': model,
+                   'session': model.Session}
+
+        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one()
+
+        pkgrevisions = model.Session.query(RevisionTableMappings.instance().PackageRevision).filter_by(id=anna1.id).all()
+        sorted_packages = sorted(pkgrevisions, key=lambda x: x.revision_timestamp)
+
+        context['revision_id'] = sorted_packages[0].revision_id  # original state
+
+        with PackageDictizeMonkeyPatch():
+            first_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+            assert self.remove_changable_columns(self.package_expected) == first_dictized
+
+            context['revision_id'] = sorted_packages[1].revision_id
+
+            second_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+
+            first_dictized["name"] = u'annakarenina_changed'
+            first_dictized["resources"][0]["url"] = u'http://new_url'
+
+            assert second_dictized == first_dictized
+
+            context['revision_id'] = sorted_packages[2].revision_id
+            third_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+
+            second_dictized['name'] = u'annakarenina_changed2'
+            second_dictized['resources'][0]['url'] = u'http://new_url2'
+            second_dictized['tags'][0]['name'] = u'new_tag'
+            second_dictized['tags'][0]['display_name'] = u'new_tag'
+            second_dictized['state'] = 'active'
+
+            print('\n'.join(unified_diff(pformat(second_dictized).split('\n'), pformat(third_dictized).split('\n'))))
+            assert second_dictized == third_dictized
+
+            context['revision_id'] = sorted_packages[3].revision_id  # original state
+            forth_dictized = self.remove_changable_columns(package_dictize(anna1, context))
+
+        third_dictized['notes'] = 'wee'
+        third_dictized['resources'].insert(2, {
+            u'cache_last_updated': None,
+            u'cache_url': None,
+            u'description': u'',
+            u'format': u'plain text',
+            u'hash': u'',
+            u'last_modified': None,
+            u'mimetype': None,
+            u'mimetype_inner': None,
+            u'name': None,
+            u'position': 2,
+            u'resource_type': None,
+            u'url_type': None,
+            u'size': None,
+            u'state': u'active',
+            u'url': u'http://newurl'})
+        third_dictized['num_resources'] = third_dictized['num_resources'] + 1
+
+        third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'display_name': u'newnew_tag', 'state': 'active'})
+        third_dictized['num_tags'] = third_dictized['num_tags'] + 1
+        third_dictized['state'] = 'active'
+        third_dictized['state'] = 'active'
+
+        pprint(third_dictized)
+        pprint(forth_dictized)
+
+        assert third_dictized == forth_dictized
+
+    def remove_changable_columns(self, dict, remove_package_id=False):
+        ids_to_keep = ['license_id', 'creator_user_id']
+        if not remove_package_id:
+            ids_to_keep.append('package_id')
+
+        for key, value in dict.items():
+            if key.endswith('id') and key not in ids_to_keep:
+                dict.pop(key)
+            if key == 'created':
+                dict.pop(key)
+            if 'timestamp' in key:
+                dict.pop(key)
+            if key in ['metadata_created','metadata_modified']:
+                dict.pop(key)
+            if isinstance(value, list):
+                for new_dict in value:
+                    self.remove_changable_columns(new_dict,
+                        key in ['resources', 'extras'] or remove_package_id)
+
+            # TEMPORARY HACK - we remove 'extras' so they aren't tested. This
+            # is due to package_extra_revisions being migrated from ckan/model
+            # in #4691 but not the rest of the model revisions just yet. Until
+            # we finish this work (#4664) it is hard to get this working -
+            # extra_revision_table is no longer being populated because
+            # PackageExtra no longer has
+            # vdm.sqlalchemy.Revisioner(extra_revision_table). However #4664
+            # will allow use to manually create revisions and test this again.
+            if key == 'extras':
+                dict.pop(key)
+            # END OF HACK
+        return dict

From 4bd483bc9880d1a12a27d99a71e5ffed92eec180 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 21 Nov 2019 20:46:32 +0100
Subject: [PATCH 06/30] [#4801] StringIO is no longer a module on py3

---
 ckan/tests/controllers/test_api.py         |  2 --
 ckan/tests/legacy/functional/api/base.py   |  7 +------
 ckan/tests/legacy/test_coding_standards.py |  6 +++---
 ckan/tests/lib/test_cli.py                 |  3 ++-
 ckan/tests/logic/action/test_create.py     | 18 ++++++++++--------
 ckan/tests/logic/action/test_update.py     | 19 +++++++++++--------
 ckanext/datastore/backend/postgres.py      |  3 +--
 7 files changed, 28 insertions(+), 30 deletions(-)

diff --git a/ckan/tests/controllers/test_api.py b/ckan/tests/controllers/test_api.py
index 01fbb978456..af73e40a653 100644
--- a/ckan/tests/controllers/test_api.py
+++ b/ckan/tests/controllers/test_api.py
@@ -39,8 +39,6 @@ class TestApiController(helpers.FunctionalTestBase):
     def test_resource_create_upload_file(self, _):
         user = factories.User()
         pkg = factories.Dataset(creator_user_id=user['id'])
-        # upload_content = StringIO()
-        # upload_content.write('test-content')
 
         url = url_for(
             controller='api',
diff --git a/ckan/tests/legacy/functional/api/base.py b/ckan/tests/legacy/functional/api/base.py
index 9f9f3062c2d..ae0007a0e20 100644
--- a/ckan/tests/legacy/functional/api/base.py
+++ b/ckan/tests/legacy/functional/api/base.py
@@ -1,18 +1,13 @@
 # encoding: utf-8
 
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
-
 from nose.tools import assert_equal
 from paste.fixture import TestRequest
 from webhelpers.html import url_escape
 
 from six.moves.urllib.parse import quote
+from six import StringIO
 
 import ckan.model as model
-from ckan.tests.legacy import CreateTestData
 from ckan.tests.legacy import TestController as ControllerTestCase
 from ckan.common import json
 
diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py
index 388c5f7599c..2250cb53e42 100644
--- a/ckan/tests/legacy/test_coding_standards.py
+++ b/ckan/tests/legacy/test_coding_standards.py
@@ -16,14 +16,14 @@
 current coding standards.  Please add comments by files that fail if there
 are legitimate reasons for the failure.
 '''
-
-import cStringIO
 import inspect
 import itertools
 import os
 import re
 import sys
 
+from six import StringIO
+
 import pycodestyle
 
 file_path = os.path.dirname(__file__)
@@ -517,7 +517,7 @@ def test_pep8_pass(self):
     @classmethod
     def find_pep8_errors(cls, filename=None, lines=None):
         try:
-            sys.stdout = cStringIO.StringIO()
+            sys.stdout = StringIO()
             config = {'ignore': [
                 # W503/W504 - breaking before/after binary operators is agreed
                 # to not be a concern and was changed to be ignored by default.
diff --git a/ckan/tests/lib/test_cli.py b/ckan/tests/lib/test_cli.py
index e8491f02be9..68910d7c64a 100644
--- a/ckan/tests/lib/test_cli.py
+++ b/ckan/tests/lib/test_cli.py
@@ -4,7 +4,6 @@
 import logging
 import os
 import os.path
-from StringIO import StringIO
 import sys
 import tempfile
 
@@ -12,6 +11,8 @@
                         assert_not_in, assert_not_equal as neq, assert_false as nok)
 from paste.script.command import run
 
+from six import StringIO
+
 import ckan.lib.cli as cli
 import ckan.lib.jobs as jobs
 import ckan.tests.helpers as helpers
diff --git a/ckan/tests/logic/action/test_create.py b/ckan/tests/logic/action/test_create.py
index 79a99e65dbd..6469fc5c131 100644
--- a/ckan/tests/logic/action/test_create.py
+++ b/ckan/tests/logic/action/test_create.py
@@ -5,7 +5,6 @@
 '''
 import __builtin__ as builtins
 
-import six
 
 import ckan
 import ckan.logic as logic
@@ -16,6 +15,9 @@
 import mock
 import nose.tools
 from ckan.common import config
+
+from six import string_types, StringIO
+
 from pyfakefs import fake_filesystem
 
 eq = assert_equals = nose.tools.assert_equals
@@ -523,8 +525,8 @@ def test_mimetype_by_upload_by_filename(self, mock_open):
         Real world usage would be using the FileStore API or web UI form to upload a file, with a filename plus extension
         If there's no url or the mimetype can't be guessed by the url, mimetype will be guessed by the extension in the filename
         '''
-        import StringIO
-        test_file = StringIO.StringIO()
+
+        test_file = StringIO()
         test_file.write('''
         "info": {
             "title": "BC Data Catalogue API",
@@ -573,8 +575,8 @@ def test_mimetype_by_upload_by_file(self, mock_open):
         Real world usage would be using the FileStore API or web UI form to upload a file, that has no extension
         If the mimetype can't be guessed by the url or filename, mimetype will be guessed by the contents inside the file
         '''
-        import StringIO
-        test_file = StringIO.StringIO()
+
+        test_file = StringIO()
         test_file.write('''
         Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm
         SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54
@@ -608,8 +610,8 @@ def test_size_of_resource_by_upload(self, mock_open):
         '''
         The size of the resource determined by the uploaded file
         '''
-        import StringIO
-        test_file = StringIO.StringIO()
+
+        test_file = StringIO()
         test_file.write('''
         Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm
         SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54
@@ -991,7 +993,7 @@ def test_return_id_only(self):
             context={'return_id_only': True},
         )
 
-        assert isinstance(dataset, six.string_types)
+        assert isinstance(dataset, string_types)
 
 
 class TestGroupCreate(helpers.FunctionalTestBase):
diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py
index 9c2200617cc..034722235e2 100644
--- a/ckan/tests/logic/action/test_update.py
+++ b/ckan/tests/logic/action/test_update.py
@@ -14,6 +14,9 @@
 import nose.tools
 from ckan import model
 from ckan.common import config
+
+
+from six import StringIO
 from pyfakefs import fake_filesystem
 
 assert_equals = eq_ = nose.tools.assert_equals
@@ -1059,8 +1062,8 @@ def test_mimetype_by_upload_by_file(self, mock_open):
                                       url='http://localhost/data.csv',
                                       name='Test')
 
-        import StringIO
-        update_file = StringIO.StringIO()
+
+        update_file = StringIO()
         update_file.write('''
         Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm
         SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54
@@ -1093,8 +1096,8 @@ def test_mimetype_by_upload_by_filename(self, mock_open):
         Real world usage would be using the FileStore API or web UI form to upload a file, with a filename plus extension
         If there's no url or the mimetype can't be guessed by the url, mimetype will be guessed by the extension in the filename
         '''
-        import StringIO
-        test_file = StringIO.StringIO()
+
+        test_file = StringIO()
         test_file.write('''
         "info": {
             "title": "BC Data Catalogue API",
@@ -1122,7 +1125,7 @@ def test_mimetype_by_upload_by_filename(self, mock_open):
                                           name='Test',
                                           upload=test_resource)
 
-        update_file = StringIO.StringIO()
+        update_file = StringIO()
         update_file.write('''
         Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm
         SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54
@@ -1173,8 +1176,8 @@ def test_size_of_resource_by_upload(self, mock_open):
         '''
         The size of the resource determined by the uploaded file
         '''
-        import StringIO
-        test_file = StringIO.StringIO()
+
+        test_file = StringIO()
         test_file.write('''
         "info": {
             "title": "BC Data Catalogue API",
@@ -1202,7 +1205,7 @@ def test_size_of_resource_by_upload(self, mock_open):
                                           name='Test',
                                           upload=test_resource)
 
-        update_file = StringIO.StringIO()
+        update_file = StringIO()
         update_file.write('''
         Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm
         SKINS LAKE,1B05,890,2015/12/30,34,53,,98,16,JAN-01,54
diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
index 6a0792ea34d..6ff211704df 100644
--- a/ckanext/datastore/backend/postgres.py
+++ b/ckanext/datastore/backend/postgres.py
@@ -10,12 +10,11 @@
 import datetime
 import hashlib
 import json
-from cStringIO import StringIO
 
 from six.moves.urllib.parse import (
     urlencode, unquote, urlunparse, parse_qsl, urlparse
 )
-from six import string_types, text_type
+from six import string_types, text_type, StringIO
 
 import ckan.lib.cli as cli
 import ckan.plugins as p

From 52e6f5e0bc6e0a8b97f3989f4adc5962090d6ac7 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 21 Nov 2019 20:52:32 +0100
Subject: [PATCH 07/30] [#4801] Add beaker as requirement (previously came via
 Pylons)

---
 requirements.in  | 1 +
 requirements.txt | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/requirements.in b/requirements.in
index 2c627dac919..ede48112b7f 100644
--- a/requirements.in
+++ b/requirements.in
@@ -2,6 +2,7 @@
 # Use pip-compile to create a requirements.txt file from this
 alembic==1.0.0
 Babel==2.3.4
+Beaker==1.11.0
 bleach==3.0.2
 click==6.7
 fanstatic==0.12
diff --git a/requirements.txt b/requirements.txt
index 7ce0cd91d4f..a2c2365872a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,6 +6,7 @@
 #
 alembic==1.0.0
 babel==2.3.4
+beaker==1.11.0
 bleach==3.0.2
 certifi==2019.9.11        # via requests
 chardet==3.0.4            # via requests
@@ -14,6 +15,7 @@ decorator==4.4.1          # via sqlalchemy-migrate
 fanstatic==0.12
 flask-babel==0.11.2
 flask==1.1.1
+funcsigs==1.0.2           # via beaker
 idna==2.8                 # via requests
 itsdangerous==1.1.0       # via flask
 jinja2==2.10.1
@@ -49,10 +51,10 @@ sqlparse==0.2.2
 tempita==0.5.2            # via sqlalchemy-migrate
 tzlocal==1.3
 unicodecsv==0.14.1
-urllib3==1.25.6           # via requests
+urllib3==1.25.7           # via requests
 webassets==0.12.1
 webencodings==0.5.1       # via bleach
-webob==1.0.8              # via fanstatic, repoze.who, webtest
+webob==1.8.5              # via fanstatic, repoze.who, webtest
 webtest==1.4.3
 werkzeug==0.15.5
 zope.interface==4.3.2

From 0b27bde48fc34bd0bd6301455461ae1b6314a2f3 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 10:59:54 +0100
Subject: [PATCH 08/30] Re-add needed import

---
 ckan/tests/legacy/functional/api/base.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ckan/tests/legacy/functional/api/base.py b/ckan/tests/legacy/functional/api/base.py
index ae0007a0e20..be6091add10 100644
--- a/ckan/tests/legacy/functional/api/base.py
+++ b/ckan/tests/legacy/functional/api/base.py
@@ -8,6 +8,7 @@
 from six import StringIO
 
 import ckan.model as model
+from ckan.tests.legacy import CreateTestData
 from ckan.tests.legacy import TestController as ControllerTestCase
 from ckan.common import json
 

From 465400c2def4d42cf3ec37a12cf14369d727f02c Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 11:18:48 +0100
Subject: [PATCH 09/30] [#4801] Refactor parse_db_config to avoid importing
 from old cli

---
 ckan/cli/datastore.py                 | 30 ++++++++----------------
 ckan/lib/cli.py                       | 33 ++++++---------------------
 ckan/model/__init__.py                | 21 +++++++++++++++++
 ckanext/datastore/backend/postgres.py |  3 +--
 ckanext/datastore/tests/helpers.py    |  5 ++--
 5 files changed, 40 insertions(+), 52 deletions(-)

diff --git a/ckan/cli/datastore.py b/ckan/cli/datastore.py
index 0320f924d75..4803777e231 100644
--- a/ckan/cli/datastore.py
+++ b/ckan/cli/datastore.py
@@ -6,7 +6,7 @@
 
 import click
 
-from ckan.cli import error_shout
+from ckan.model import parse_db_config
 from ckan.common import config
 
 import ckanext.datastore as datastore_module
@@ -30,9 +30,9 @@ def set_permissions():
     u'''Emit an SQL script that will set the permissions for the datastore
     users as configured in your configuration file.'''
 
-    write_url = parse_db_config(u'ckan.datastore.write_url')
-    read_url = parse_db_config(u'ckan.datastore.read_url')
-    db_url = parse_db_config(u'sqlalchemy.url')
+    write_url = _parse_db_config(u'ckan.datastore.write_url')
+    read_url = _parse_db_config(u'ckan.datastore.read_url')
+    db_url = _parse_db_config(u'sqlalchemy.url')
 
     # Basic validation that read and write URLs reference the same database.
     # This obviously doesn't check they're the same database (the hosts/ports
@@ -102,25 +102,13 @@ def dump(ctx, resource_id, output_file, format, offset, limit, bom):
         )
 
 
-def parse_db_config(config_key=u'sqlalchemy.url'):
-    u''' Takes a config key for a database connection url and parses it into
-    a dictionary. Expects a url like:
-
-    'postgres://tester:pass@localhost/ckantest3'
-    '''
-    url = config[config_key]
-    regex = [
-        u'^\\s*(?P<db_type>\\w*)', u'://', u'(?P<db_user>[^:]*)', u':?',
-        u'(?P<db_pass>[^@]*)', u'@', u'(?P<db_host>[^/:]*)', u':?',
-        u'(?P<db_port>[^/]*)', u'/', u'(?P<db_name>[\\w.-]*)'
-    ]
-    db_details_match = re.match(u''.join(regex), url)
-    if not db_details_match:
+def _parse_db_config(config_key=u'sqlalchemy.url'):
+    db_config = parse_db_config(config_key)
+    if not db_config:
         click.secho(
-            u'Could not extract db details from url: %r' % url,
+            u'Could not extract db details from url: %r' % config[config_key],
             fg=u'red',
             bold=True
         )
         raise click.Abort()
-    db_details = db_details_match.groupdict()
-    return db_details
+    return db_config
diff --git a/ckan/lib/cli.py b/ckan/lib/cli.py
index dd826e8a1ab..9f1271fda00 100644
--- a/ckan/lib/cli.py
+++ b/ckan/lib/cli.py
@@ -70,32 +70,13 @@ def error(msg):
     sys.exit(1)
 
 
-def parse_db_config(config_key='sqlalchemy.url'):
-    ''' Takes a config key for a database connection url and parses it into
-    a dictionary. Expects a url like:
-
-    'postgres://tester:pass@localhost/ckantest3'
-    '''
-    from ckan.common import config
-    url = config[config_key]
-    regex = [
-        '^\s*(?P<db_type>\w*)',
-        '://',
-        '(?P<db_user>[^:]*)',
-        ':?',
-        '(?P<db_pass>[^@]*)',
-        '@',
-        '(?P<db_host>[^/:]*)',
-        ':?',
-        '(?P<db_port>[^/]*)',
-        '/',
-        '(?P<db_name>[\w.-]*)'
-    ]
-    db_details_match = re.match(''.join(regex), url)
-    if not db_details_match:
-        raise Exception('Could not extract db details from url: %r' % url)
-    db_details = db_details_match.groupdict()
-    return db_details
+def _parse_db_config(config_key=u'sqlalchemy.url'):
+    db_config = model.parse_db_config(config_key)
+    if not db_config:
+        raise Exception(
+            u'Could not extract db details from url: %r' % config[config_key]
+        )
+    return db_config
 
 
 def user_add(args):
diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py
index e69a413be8e..027e33920af 100644
--- a/ckan/model/__init__.py
+++ b/ckan/model/__init__.py
@@ -127,6 +127,7 @@
 )
 
 import ckan.migration
+from ckan.common import config
 
 
 log = logging.getLogger(__name__)
@@ -335,3 +336,23 @@ def is_id(id_string):
     '''Tells the client if the string looks like a revision id or not'''
     reg_ex = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
     return bool(re.match(reg_ex, id_string))
+
+
+def parse_db_config(config_key=u'sqlalchemy.url'):
+    u''' Takes a config key for a database connection url and parses it into
+    a dictionary. Expects a url like:
+
+    'postgres://tester:pass@localhost/ckantest3'
+
+    Returns None if the url could not be parsed.
+    '''
+    url = config[config_key]
+    regex = [
+        u'^\\s*(?P<db_type>\\w*)', u'://', u'(?P<db_user>[^:]*)', u':?',
+        u'(?P<db_pass>[^@]*)', u'@', u'(?P<db_host>[^/:]*)', u':?',
+        u'(?P<db_port>[^/]*)', u'/', u'(?P<db_name>[\\w.-]*)'
+    ]
+    db_details_match = re.match(u''.join(regex), url)
+    if not db_details_match:
+        return
+    return db_details_match.groupdict()
diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
index 6ff211704df..e7f93894815 100644
--- a/ckanext/datastore/backend/postgres.py
+++ b/ckanext/datastore/backend/postgres.py
@@ -16,7 +16,6 @@
 )
 from six import string_types, text_type, StringIO
 
-import ckan.lib.cli as cli
 import ckan.plugins as p
 import ckan.plugins.toolkit as toolkit
 from ckan.lib.lazyjson import LazyJSONObject
@@ -330,7 +329,7 @@ def _pg_version_is_at_least(connection, version):
 
 
 def _get_read_only_user(data_dict):
-    parsed = cli.parse_db_config('ckan.datastore.read_url')
+    parsed = model.parse_db_config('ckan.datastore.read_url')
     return parsed['db_user']
 
 
diff --git a/ckanext/datastore/tests/helpers.py b/ckanext/datastore/tests/helpers.py
index b5fa01df802..3464383e67a 100644
--- a/ckanext/datastore/tests/helpers.py
+++ b/ckanext/datastore/tests/helpers.py
@@ -3,7 +3,6 @@
 from sqlalchemy import orm
 
 import ckan.model as model
-import ckan.lib.cli as cli
 from ckan.lib import search
 
 import ckan.plugins as p
@@ -41,8 +40,8 @@ def rebuild_all_dbs(Session):
     ''' If the tests are running on the same db, we have to make sure that
     the ckan tables are recrated.
     '''
-    db_read_url_parts = cli.parse_db_config('ckan.datastore.write_url')
-    db_ckan_url_parts = cli.parse_db_config('sqlalchemy.url')
+    db_read_url_parts = model.parse_db_config('ckan.datastore.write_url')
+    db_ckan_url_parts = model.parse_db_config('sqlalchemy.url')
     same_db = db_read_url_parts['db_name'] == db_ckan_url_parts['db_name']
 
     if same_db:

From 472663aa4648e58f2c2b6b99604b68118b11c5fd Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 12:15:37 +0100
Subject: [PATCH 10/30] [#4801] Selective loading in plugins toolkit

This took a while to get right because exceptions raised during the
toolkit initialization fail silently and the sympton would be the an
import error later on when trying to import something from the toolkit:

from ckan.plugins.toolkit import get_action

ImportError: cannot import name 'get_action'

TODO: Sort out what gets exposed to extensions in terms of CLI commands
---
 ckan/plugins/toolkit.py | 42 +++++++++++++++++++++++------------------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index c2874421e2b..04bc9212401 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -130,14 +130,14 @@ def __init__(self):
     def _initialize(self):
         ''' get the required functions/objects, store them for later
         access and check that they match the contents dict. '''
-
+        import six
         import ckan
-        import ckan.lib.base as base
         import ckan.logic as logic
+
+        import ckan.lib.base as base
         import ckan.logic.validators as logic_validators
         import ckan.lib.navl.dictization_functions as dictization_functions
         import ckan.lib.helpers as h
-        import ckan.lib.cli as cli
         import ckan.lib.plugins as lib_plugins
         import ckan.common as common
         from ckan.exceptions import (
@@ -147,8 +147,10 @@ def _initialize(self):
         from ckan.lib.jobs import enqueue as enqueue_job
 
         import ckan.common as converters
-        import pylons
-        import webhelpers.html.tags
+        if six.PY2:
+            import ckan.lib.cli as cli
+            import pylons
+            import webhelpers.html.tags
 
         # Allow class access to these modules
         self.__class__.ckan = ckan
@@ -213,6 +215,7 @@ def _initialize(self):
 
 '''
         t['render'] = base.render
+        t['abort'] = base.abort
         t['asbool'] = converters.asbool
         self.docstring_overrides['asbool'] = '''Convert a string (e.g. 1,
 true, True) from the config file into a boolean.
@@ -234,7 +237,6 @@ def _initialize(self):
 For example: ``bar = toolkit.aslist(config.get('ckan.foo.bar', []))``
 
 '''
-        t['literal'] = webhelpers.html.tags.literal
 
         t['get_action'] = logic.get_action
         t['chained_action'] = logic.chained_action
@@ -251,22 +253,10 @@ def _initialize(self):
         t['UnknownValidator'] = logic.UnknownValidator
         t['Invalid'] = logic_validators.Invalid
 
-        t['CkanCommand'] = cli.CkanCommand
-        t['load_config'] = cli.load_config
         t['DefaultDatasetForm'] = lib_plugins.DefaultDatasetForm
         t['DefaultGroupForm'] = lib_plugins.DefaultGroupForm
         t['DefaultOrganizationForm'] = lib_plugins.DefaultOrganizationForm
 
-        t['response'] = pylons.response
-        self.docstring_overrides['response'] = '''The Pylons response object.
-
-Pylons uses this object to generate the HTTP response it returns to the web
-browser. It has attributes like the HTTP status code, the response headers,
-content type, cookies, etc.
-
-'''
-        t['BaseController'] = base.BaseController
-        t['abort'] = base.abort
         t['redirect_to'] = h.redirect_to
         t['url_for'] = h.url_for
         t['get_or_bust'] = logic.get_or_bust
@@ -290,6 +280,22 @@ def _initialize(self):
         t['HelperError'] = HelperError
         t['enqueue_job'] = enqueue_job
 
+        if six.PY2:
+
+            t['literal'] = webhelpers.html.tags.literal
+            t['response'] = pylons.response
+            self.docstring_overrides['response'] = '''The Pylons response object.
+
+Pylons uses this object to generate the HTTP response it returns to the web
+browser. It has attributes like the HTTP status code, the response headers,
+content type, cookies, etc.
+
+'''
+            t['BaseController'] = base.BaseController
+            # TODO: Sort these out
+            t['CkanCommand'] = cli.CkanCommand
+            t['load_config'] = cli.load_config
+
         # check contents list correct
         errors = set(t).symmetric_difference(set(self.contents))
         if errors:

From 40895ddce7803499e813e6a2b4f87bf5da07b7d8 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 12:32:55 +0100
Subject: [PATCH 11/30] [#4801] Use only Flask WSGI app in py3

---
 ckan/config/middleware/__init__.py | 58 +++++++++++++++---------------
 1 file changed, 30 insertions(+), 28 deletions(-)

diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py
index 92fc6e3d2a6..012baf40f88 100644
--- a/ckan/config/middleware/__init__.py
+++ b/ckan/config/middleware/__init__.py
@@ -2,44 +2,42 @@
 
 """WSGI app initialization"""
 
-import webob
-from routes import request_config as routes_request_config
-
+import logging
+import six
 from six.moves.urllib.parse import urlparse, quote
 
 from ckan.lib.i18n import get_locales_from_config
 from ckan.config.environment import load_environment
 from ckan.config.middleware.flask_app import make_flask_stack
-from ckan.config.middleware.pylons_app import make_pylons_stack
 from ckan.common import config
 
-from ckan.lib.i18n import get_locales_from_config
-
-import logging
 log = logging.getLogger(__name__)
 
-# This monkey-patches the webob request object because of the way it messes
-# with the WSGI environ.
-
-# Start of webob.requests.BaseRequest monkey patch
-original_charset__set = webob.request.BaseRequest._charset__set
+if six.PY2:
+    import webob
+    from routes import request_config as routes_request_config
+    from ckan.config.middleware.pylons_app import make_pylons_stack
 
+    # This monkey-patches the webob request object because of the way it messes
+    # with the WSGI environ.
 
-def custom_charset__set(self, charset):
-    original_charset__set(self, charset)
-    if self.environ.get('CONTENT_TYPE', '').startswith(';'):
-        self.environ['CONTENT_TYPE'] = ''
+    # Start of webob.requests.BaseRequest monkey patch
+    original_charset__set = webob.request.BaseRequest._charset__set
 
+    def custom_charset__set(self, charset):
+        original_charset__set(self, charset)
+        if self.environ.get('CONTENT_TYPE', '').startswith(';'):
+            self.environ['CONTENT_TYPE'] = ''
 
-webob.request.BaseRequest._charset__set = custom_charset__set
+    webob.request.BaseRequest._charset__set = custom_charset__set
 
-webob.request.BaseRequest.charset = property(
-    webob.request.BaseRequest._charset__get,
-    custom_charset__set,
-    webob.request.BaseRequest._charset__del,
-    webob.request.BaseRequest._charset__get.__doc__)
+    webob.request.BaseRequest.charset = property(
+        webob.request.BaseRequest._charset__get,
+        custom_charset__set,
+        webob.request.BaseRequest._charset__del,
+        webob.request.BaseRequest._charset__get.__doc__)
 
-# End of webob.requests.BaseRequest monkey patch
+    # End of webob.requests.BaseRequest monkey patch
 
 # This is a test Flask request context to be used internally.
 # Do not use it!
@@ -54,12 +52,16 @@ def make_app(conf, full_stack=True, static_files=True, **app_conf):
 
     load_environment(conf, app_conf)
 
-    pylons_app = make_pylons_stack(conf, full_stack, static_files,
-                                   **app_conf)
     flask_app = make_flask_stack(conf, **app_conf)
-
-    app = AskAppDispatcherMiddleware({'pylons_app': pylons_app,
-                                      'flask_app': flask_app})
+    if six.PY2:
+        pylons_app = make_pylons_stack(
+            conf, full_stack, static_files, **app_conf)
+
+        app = AskAppDispatcherMiddleware(
+            {'pylons_app': pylons_app,
+             'flask_app': flask_app})
+    else:
+        app = flask_app
 
     # Set this internal test request context with the configured environment so
     # it can be used when calling url_for from tests

From 3ed55d6530b8a6b95cf9ced6341e3df132f54f1f Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 12:33:23 +0100
Subject: [PATCH 12/30] [#4801] izip_longest renamed in py3

---
 ckanext/datastore/blueprint.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ckanext/datastore/blueprint.py b/ckanext/datastore/blueprint.py
index 815d6af7e6c..d462974707e 100644
--- a/ckanext/datastore/blueprint.py
+++ b/ckanext/datastore/blueprint.py
@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-from itertools import izip_longest
+from six.moves import zip_longest
 
 from flask import Blueprint, make_response
 from flask.views import MethodView
@@ -143,7 +143,7 @@ def post(self, id, resource_id):
                     u'id': f[u'id'],
                     u'type': f[u'type'],
                     u'info': fi if isinstance(fi, dict) else {}
-                } for f, fi in izip_longest(fields, info)]
+                } for f, fi in zip_longest(fields, info)]
             }
         )
 

From 14540fc235c8ae8ad8b6e09329861d1868061f76 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 12:47:02 +0100
Subject: [PATCH 13/30] [#4801] Don't proxy keys to pylons config on py3

---
 ckan/common.py | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/ckan/common.py b/ckan/common.py
index 5c9f61aa8da..1143439bcf1 100644
--- a/ckan/common.py
+++ b/ckan/common.py
@@ -124,12 +124,14 @@ def clear(self):
             flask.current_app.config.clear()
         except RuntimeError:
             pass
-        try:
-            pylons.config.clear()
-            # Pylons set this default itself
-            pylons.config[u'lang'] = None
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                pylons.config.clear()
+                # Pylons set this default itself
+                pylons.config[u'lang'] = None
+            except TypeError:
+                pass
 
     def __setitem__(self, key, value):
         self.store[key] = value
@@ -137,10 +139,12 @@ def __setitem__(self, key, value):
             flask.current_app.config[key] = value
         except RuntimeError:
             pass
-        try:
-            pylons.config[key] = value
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                pylons.config[key] = value
+            except TypeError:
+                pass
 
     def __delitem__(self, key):
         del self.store[key]
@@ -148,10 +152,12 @@ def __delitem__(self, key):
             del flask.current_app.config[key]
         except RuntimeError:
             pass
-        try:
-            del pylons.config[key]
-        except TypeError:
-            pass
+
+        if six.PY2:
+            try:
+                del pylons.config[key]
+            except TypeError:
+                pass
 
 
 def _get_request():

From da100c82ce4da90f8267879ef11789c65d0da8d8 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 12:47:32 +0100
Subject: [PATCH 14/30] [#4801] Selective execution in environment.py

---
 ckan/config/environment.py | 93 ++++++++++++++++++++------------------
 1 file changed, 49 insertions(+), 44 deletions(-)

diff --git a/ckan/config/environment.py b/ckan/config/environment.py
index 678226febdb..85f65c0fe5b 100644
--- a/ckan/config/environment.py
+++ b/ckan/config/environment.py
@@ -45,35 +45,36 @@ def load_environment(global_conf, app_conf):
     Configure the Pylons environment via the ``pylons.config`` object. This
     code should only need to be run once.
     """
-    # this must be run at a time when the env is semi-setup, thus inlined here.
-    # Required by the deliverance plugin and iATI
-    from pylons.wsgiapp import PylonsApp
-    import pkg_resources
-    find_controller_generic = getattr(
-        PylonsApp.find_controller,
-        '_old_find_controller',
-        PylonsApp.find_controller)
-
-    # This is from pylons 1.0 source, will monkey-patch into 0.9.7
-    def find_controller(self, controller):
-        if controller in self.controller_classes:
-            return self.controller_classes[controller]
-        # Check to see if its a dotted name
-        if '.' in controller or ':' in controller:
-            ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
-
-            if hasattr(ep, 'resolve'):
-                # setuptools >= 10.2
-                mycontroller = ep.resolve()
-            else:
-                # setuptools >= 11.3
-                mycontroller = ep.load(False)
-
-            self.controller_classes[controller] = mycontroller
-            return mycontroller
-        return find_controller_generic(self, controller)
-    find_controller._old_find_controller = find_controller_generic
-    PylonsApp.find_controller = find_controller
+    if six.PY2:
+        # this must be run at a time when the env is semi-setup, thus inlined
+        # here. Required by the deliverance plugin and iATI
+        from pylons.wsgiapp import PylonsApp
+        import pkg_resources
+        find_controller_generic = getattr(
+            PylonsApp.find_controller,
+            '_old_find_controller',
+            PylonsApp.find_controller)
+
+        # This is from pylons 1.0 source, will monkey-patch into 0.9.7
+        def find_controller(self, controller):
+            if controller in self.controller_classes:
+                return self.controller_classes[controller]
+            # Check to see if its a dotted name
+            if '.' in controller or ':' in controller:
+                ep = pkg_resources.EntryPoint.parse('x={0}'.format(controller))
+
+                if hasattr(ep, 'resolve'):
+                    # setuptools >= 10.2
+                    mycontroller = ep.resolve()
+                else:
+                    # setuptools >= 11.3
+                    mycontroller = ep.load(False)
+
+                self.controller_classes[controller] = mycontroller
+                return mycontroller
+            return find_controller_generic(self, controller)
+        find_controller._old_find_controller = find_controller_generic
+        PylonsApp.find_controller = find_controller
 
     os.environ['CKAN_CONFIG'] = global_conf['__file__']
 
@@ -235,20 +236,23 @@ def update_config():
                              config.get('solr_password'))
     search.check_solr_schema_version()
 
-    routes_map = routing.make_map()
+    if six.PY2:
+        routes_map = routing.make_map()
 
     lib_plugins.reset_package_plugins()
     lib_plugins.register_package_plugins()
     lib_plugins.reset_group_plugins()
     lib_plugins.register_group_plugins()
 
-    config['routes.map'] = routes_map
-    # The RoutesMiddleware needs its mapper updating if it exists
-    if 'routes.middleware' in config:
-        config['routes.middleware'].mapper = routes_map
-    # routes.named_routes is a CKAN thing
-    config['routes.named_routes'] = routing.named_routes
-    config['pylons.app_globals'] = app_globals.app_globals
+    if six.PY2:
+        config['routes.map'] = routes_map
+        # The RoutesMiddleware needs its mapper updating if it exists
+        if 'routes.middleware' in config:
+            config['routes.middleware'].mapper = routes_map
+        # routes.named_routes is a CKAN thing
+        config['routes.named_routes'] = routing.named_routes
+        config['pylons.app_globals'] = app_globals.app_globals
+
     # initialise the globals
     app_globals.app_globals._init()
 
@@ -281,13 +285,14 @@ def update_config():
     # root logger.
     logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
 
-    # Create Jinja2 environment
-    env = jinja_extensions.Environment(
-        **jinja_extensions.get_jinja_env_options())
-    env.install_gettext_callables(_, ungettext, newstyle=True)
-    # custom filters
-    env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
-    config['pylons.app_globals'].jinja_env = env
+    if six.PY2:
+        # Create Jinja2 environment
+        env = jinja_extensions.Environment(
+            **jinja_extensions.get_jinja_env_options())
+        env.install_gettext_callables(_, ungettext, newstyle=True)
+        # custom filters
+        env.filters['empty_and_escape'] = jinja_extensions.empty_and_escape
+        config['pylons.app_globals'].jinja_env = env
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
     # any Pylons config options)

From ce577f295ac15277aac935363c1a8e961681ce1e Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 12:49:35 +0100
Subject: [PATCH 15/30] [#4801] Normalize email module name

---
 ckan/lib/mailer.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/ckan/lib/mailer.py b/ckan/lib/mailer.py
index 7f824899f6f..c3b47ecdb5c 100644
--- a/ckan/lib/mailer.py
+++ b/ckan/lib/mailer.py
@@ -5,11 +5,10 @@
 import smtplib
 import socket
 import logging
-import uuid
 from time import time
 from email.mime.text import MIMEText
 from email.header import Header
-from email import Utils
+from email import utils
 
 from ckan.common import config
 import ckan.common
@@ -49,7 +48,7 @@ def _mail_recipient(recipient_name, recipient_email,
     msg['From'] = _("%s <%s>") % (sender_name, mail_from)
     recipient = u"%s <%s>" % (recipient_name, recipient_email)
     msg['To'] = Header(recipient, 'utf-8')
-    msg['Date'] = Utils.formatdate(time())
+    msg['Date'] = utils.formatdate(time())
     msg['X-Mailer'] = "CKAN %s" % ckan.__version__
     if reply_to and reply_to != '':
         msg['Reply-to'] = reply_to

From 1a0e96515ed8b75866bf459a062c386744205467 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 13:03:15 +0100
Subject: [PATCH 16/30] [#4801] Avoid auto-importing proxies like request

As otherwise Flask will raise a RuntimeError exceptions on startup as
they are not in the context of a web request.
---
 ckan/logic/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py
index 3923b9711ae..1ac7a23f88a 100644
--- a/ckan/logic/__init__.py
+++ b/ckan/logic/__init__.py
@@ -5,6 +5,7 @@
 import re
 from collections import defaultdict
 
+from werkzeug.local import LocalProxy
 from six import string_types, text_type
 
 import ckan.model as model
@@ -392,7 +393,7 @@ def get_action(action):
         for part in module_path.split('.')[1:]:
             module = getattr(module, part)
         for k, v in module.__dict__.items():
-            if not k.startswith('_'):
+            if not k.startswith('_') and not isinstance(v, LocalProxy):
                 # Only load functions from the action module or already
                 # replaced functions.
                 if (hasattr(v, '__call__') and

From fe4f38771b329645f94c3cdbeee07438003ae4eb Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 13:27:31 +0100
Subject: [PATCH 17/30] pep8 et al

---
 ckan/lib/helpers.py                        | 12 ++++++------
 ckan/lib/i18n.py                           |  1 -
 ckan/plugins/toolkit.py                    |  3 ++-
 ckan/tests/legacy/test_coding_standards.py |  1 +
 ckan/tests/logic/action/test_update.py     |  1 -
 ckan/tests/test_coding_standards.py        |  1 +
 6 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index 3335ef9d76c..395a6dbabba 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -62,7 +62,6 @@
     from routes import url_for as _routes_default_url_for
 
 
-
 log = logging.getLogger(__name__)
 
 DEFAULT_FACET_NAMES = u'organization groups tags res_format license_id'
@@ -1368,12 +1367,13 @@ def pager_url(page, partial=None, **kwargs):
 # TODO: remove once #4794 is done
 if six.PY2:
     class Page(paginate.Page):
-        # Curry the pager method of the webhelpers.paginate.Page class, so we have
-        # our custom layout set as default.
+        # Curry the pager method of the webhelpers.paginate.Page class, so we
+        # have our custom layout set as default.
 
         def pager(self, *args, **kwargs):
             kwargs.update(
-                format=u"<div class='pagination-wrapper'><ul class='pagination'>"
+                format=u"<div class='pagination-wrapper'>"
+                "<ul class='pagination'>"
                 "$link_previous ~2~ $link_next</ul></div>",
                 symbol_previous=u'«', symbol_next=u'»',
                 curpage_attr={'class': 'active'}, link_attr={}
@@ -1401,8 +1401,8 @@ def _range(self, regexp_match):
             # Convert current page
             text = '%s' % self.page
             current_page_span = str(HTML.span(c=text, **self.curpage_attr))
-            current_page_link = self._pagerlink(self.page, text,
-                                                extra_attributes=self.curpage_attr)
+            current_page_link = self._pagerlink(
+                self.page, text, extra_attributes=self.curpage_attr)
             return re.sub(current_page_span, current_page_link, html)
 
 
diff --git a/ckan/lib/i18n.py b/ckan/lib/i18n.py
index 1655facf9b2..acc3a38f1fe 100644
--- a/ckan/lib/i18n.py
+++ b/ckan/lib/i18n.py
@@ -60,7 +60,6 @@
     import pylons
 
 
-
 log = logging.getLogger(__name__)
 
 # Default Portuguese language to Brazilian territory, since
diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index 04bc9212401..68e4e935cab 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -284,7 +284,8 @@ def _initialize(self):
 
             t['literal'] = webhelpers.html.tags.literal
             t['response'] = pylons.response
-            self.docstring_overrides['response'] = '''The Pylons response object.
+            self.docstring_overrides['response'] = '''
+The Pylons response object.
 
 Pylons uses this object to generate the HTTP response it returns to the web
 browser. It has attributes like the HTTP status code, the response headers,
diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py
index 2250cb53e42..1d2cc2ac4e2 100644
--- a/ckan/tests/legacy/test_coding_standards.py
+++ b/ckan/tests/legacy/test_coding_standards.py
@@ -456,6 +456,7 @@ class TestPep8(object):
         'ckan/tests/legacy/schema/test_schema.py',
         'ckan/tests/legacy/test_plugins.py',
         'ckan/tests/legacy/test_versions.py',
+        'ckan/tests/migration/revision_legacy_code_tests.py',
         'test_revision_legacy_code.py',
         'ckan/websetup.py',
         'ckanext/datastore/bin/datastore_setup.py',
diff --git a/ckan/tests/logic/action/test_update.py b/ckan/tests/logic/action/test_update.py
index 034722235e2..1b3669a1706 100644
--- a/ckan/tests/logic/action/test_update.py
+++ b/ckan/tests/logic/action/test_update.py
@@ -1062,7 +1062,6 @@ def test_mimetype_by_upload_by_file(self, mock_open):
                                       url='http://localhost/data.csv',
                                       name='Test')
 
-
         update_file = StringIO()
         update_file.write('''
         Snow Course Name, Number, Elev. metres, Date of Survey, Snow Depth cm, Water Equiv. mm, Survey Code, % of Normal, Density %, Survey Period, Normal mm
diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py
index 5256ef829df..4124bdf33c0 100644
--- a/ckan/tests/test_coding_standards.py
+++ b/ckan/tests/test_coding_standards.py
@@ -562,6 +562,7 @@ def find_unprefixed_string_literals(filename):
     u'ckan/tests/logic/test_schema.py',
     u'ckan/tests/logic/test_validators.py',
     u'ckan/tests/migration/__init__.py',
+    u'ckan/tests/migration/revision_legacy_code_tests.py',
     u'test_revision_legacy_code.py',
     u'ckan/tests/model/__init__.py',
     u'ckan/tests/model/test_license.py',

From 19bf78ca08fb3beda3c6cc728ce5b4cba8247403 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 22 Nov 2019 13:31:04 +0100
Subject: [PATCH 18/30] Fix wrong import from 465400

---
 ckanext/datastore/commands.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ckanext/datastore/commands.py b/ckanext/datastore/commands.py
index 2d27f30f52f..c4bff8d04a9 100644
--- a/ckanext/datastore/commands.py
+++ b/ckanext/datastore/commands.py
@@ -4,7 +4,7 @@
 
 from ckan.lib.cli import (
     load_config,
-    parse_db_config,
+    _parse_db_config,
     paster_click_group,
     click_config_option,
 )
@@ -28,9 +28,9 @@
 def set_permissions(ctx, config):
     load_config(config or ctx.obj['config'])
 
-    write_url = parse_db_config(u'ckan.datastore.write_url')
-    read_url = parse_db_config(u'ckan.datastore.read_url')
-    db_url = parse_db_config(u'sqlalchemy.url')
+    write_url = _parse_db_config(u'ckan.datastore.write_url')
+    read_url = _parse_db_config(u'ckan.datastore.read_url')
+    db_url = _parse_db_config(u'sqlalchemy.url')
 
     # Basic validation that read and write URLs reference the same database.
     # This obviously doesn't check they're the same database (the hosts/ports

From e35235770c5d146f07a1311779e768ea8fc76a81 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Wed, 27 Nov 2019 21:00:14 +0100
Subject: [PATCH 19/30] [#4801] Replace usage of iteritems() with
 six.iteritems()

---
 ckan/authz.py                              |  5 +++--
 ckan/cli/translation.py                    |  3 ++-
 ckan/config/middleware/__init__.py         |  2 +-
 ckan/config/middleware/flask_app.py        |  6 +++---
 ckan/controllers/package.py                |  3 ++-
 ckan/i18n/check_po_files.py                |  4 +++-
 ckan/lib/base.py                           |  2 +-
 ckan/lib/dictization/__init__.py           |  5 +++--
 ckan/lib/dictization/model_dictize.py      |  4 ++--
 ckan/lib/dictization/model_save.py         | 13 ++++++-------
 ckan/lib/helpers.py                        |  2 +-
 ckan/lib/i18n.py                           |  4 ++--
 ckan/lib/lazyjson.py                       |  1 +
 ckan/lib/navl/dictization_functions.py     |  7 ++++---
 ckan/lib/navl/validators.py                |  3 ++-
 ckan/lib/search/index.py                   |  2 +-
 ckan/logic/__init__.py                     |  9 +++++----
 ckan/logic/action/__init__.py              |  4 +++-
 ckan/logic/action/update.py                |  5 ++---
 ckan/logic/converters.py                   |  5 +++--
 ckan/migration/revision_legacy_code.py     |  4 ++--
 ckan/tests/controllers/test_template.py    |  3 ++-
 ckan/tests/legacy/test_coding_standards.py |  5 +++--
 ckan/tests/test_common.py                  |  4 +++-
 ckan/views/api.py                          |  5 +++--
 ckan/views/dataset.py                      |  9 ++++-----
 ckan/views/feed.py                         |  5 +++--
 ckan/views/group.py                        |  4 +++-
 ckan/views/resource.py                     |  3 ++-
 ckanext/datastore/backend/postgres.py      |  9 +++++----
 ckanext/multilingual/plugin.py             |  5 +++--
 ckanext/reclineview/plugin.py              |  4 +++-
 ckanext/textview/plugin.py                 |  4 +++-
 33 files changed, 89 insertions(+), 64 deletions(-)

diff --git a/ckan/authz.py b/ckan/authz.py
index 9c6f007acc7..f96c86c30c8 100644
--- a/ckan/authz.py
+++ b/ckan/authz.py
@@ -2,11 +2,12 @@
 
 import functools
 import sys
-import re
 
 from collections import defaultdict
 from logging import getLogger
 
+import six
+
 from ckan.common import config
 from ckan.common import asbool
 
@@ -101,7 +102,7 @@ def _build(self):
                     resolved_auth_function_plugins[name] = plugin.name
                     fetched_auth_functions[name] = auth_function
 
-        for name, func_list in chained_auth_functions.iteritems():
+        for name, func_list in six.iteritems(chained_auth_functions):
             if (name not in fetched_auth_functions and
                     name not in self._functions):
                 raise Exception('The auth %r is not found for chained auth' % (
diff --git a/ckan/cli/translation.py b/ckan/cli/translation.py
index b720192f60e..cc4a1ff3cb5 100644
--- a/ckan/cli/translation.py
+++ b/ckan/cli/translation.py
@@ -6,6 +6,7 @@
 import os
 
 import click
+import six
 
 from ckan.cli import error_shout
 from ckan.common import config
@@ -145,7 +146,7 @@ def check_po_file(path):
             for function in (
                 simple_conv_specs, mapping_keys, replacement_fields
             ):
-                for key, msgstr in entry.msgstr_plural.iteritems():
+                for key, msgstr in six.iteritems(entry.msgstr_plural):
                     if key == u'0':
                         error = check_translation(
                             function, entry.msgid, entry.msgstr_plural[key]
diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py
index 012baf40f88..68ac60f3e14 100644
--- a/ckan/config/middleware/__init__.py
+++ b/ckan/config/middleware/__init__.py
@@ -105,7 +105,7 @@ def ask_around(self, environ):
         '''
         answers = [
             app._wsgi_app.can_handle_request(environ)
-            for name, app in self.apps.iteritems()
+            for name, app in six.iteritems(self.apps)
         ]
         # Sort answers by app name
         answers = sorted(answers, key=lambda x: x[1])
diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py
index e78577c6e81..8419507d5a6 100644
--- a/ckan/config/middleware/flask_app.py
+++ b/ckan/config/middleware/flask_app.py
@@ -12,6 +12,7 @@
 from flask.ctx import _AppCtxGlobals
 from flask.sessions import SessionInterface
 
+import six
 from werkzeug.exceptions import default_exceptions, HTTPException
 from werkzeug.routing import Rule
 
@@ -23,7 +24,6 @@
 from repoze.who.config import WhoConfig
 from repoze.who.middleware import PluggableAuthenticationMiddleware
 
-import ckan
 import ckan.model as model
 from ckan.lib import base
 from ckan.lib import helpers
@@ -130,7 +130,7 @@ def save_session(self, app, session, response):
 
     namespace = 'beaker.session.'
     session_opts = {k.replace('beaker.', ''): v
-                    for k, v in config.iteritems()
+                    for k, v in six.iteritems(config)
                     if k.startswith(namespace)}
     if (not session_opts.get('session.data_dir') and
             session_opts.get('session.type', 'file') == 'file'):
@@ -417,7 +417,7 @@ def register_extension_blueprint(self, blueprint, **kwargs):
 
         # Get the new blueprint rules
         bp_rules = itertools.chain.from_iterable(
-            v for k, v in self.url_map._rules_by_endpoint.iteritems()
+            v for k, v in six.iteritems(self.url_map._rules_by_endpoint)
             if k.startswith(u'{0}.'.format(blueprint.name))
         )
 
diff --git a/ckan/controllers/package.py b/ckan/controllers/package.py
index 71e40a1d287..47c6ddfb042 100644
--- a/ckan/controllers/package.py
+++ b/ckan/controllers/package.py
@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 import logging
+import six
 from six.moves.urllib.parse import urlencode
 import datetime
 import mimetypes
@@ -648,7 +649,7 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
 
             # see if we have any data that we are trying to save
             data_provided = False
-            for key, value in data.iteritems():
+            for key, value in six.iteritems(data):
                 if ((value or isinstance(value, cgi.FieldStorage))
                         and key != 'resource_type'):
                     data_provided = True
diff --git a/ckan/i18n/check_po_files.py b/ckan/i18n/check_po_files.py
index a3acb09ef5f..6f652fa03cc 100755
--- a/ckan/i18n/check_po_files.py
+++ b/ckan/i18n/check_po_files.py
@@ -11,6 +11,8 @@
 import re
 import paste.script.command
 
+import six
+
 
 def simple_conv_specs(s):
     '''Return the simple Python string conversion specifiers in the string s.
@@ -79,7 +81,7 @@ def check_translation(validator, msgid, msgstr):
         if entry.msgid_plural and entry.msgstr_plural:
             for function in (simple_conv_specs, mapping_keys,
                              replacement_fields):
-                for key, msgstr in entry.msgstr_plural.iteritems():
+                for key, msgstr in six.iteritems(entry.msgstr_plural):
                     if key == '0':
                         check_translation(function, entry.msgid,
                                           entry.msgstr_plural[key])
diff --git a/ckan/lib/base.py b/ckan/lib/base.py
index d73b98446c3..43877e50cc2 100644
--- a/ckan/lib/base.py
+++ b/ckan/lib/base.py
@@ -217,7 +217,7 @@ def set_pylons_response_headers(allow_cache):
         allow_cache = False
     # Don't cache if we have extra vars containing data.
     elif extra_vars:
-        for k, v in extra_vars.iteritems():
+        for k, v in six.iteritems(extra_vars):
             allow_cache = False
             break
 
diff --git a/ckan/lib/dictization/__init__.py b/ckan/lib/dictization/__init__.py
index 0a5a73cea8f..6e1f219b6bc 100644
--- a/ckan/lib/dictization/__init__.py
+++ b/ckan/lib/dictization/__init__.py
@@ -3,8 +3,9 @@
 import datetime
 from sqlalchemy.orm import class_mapper
 import sqlalchemy
+
+import six
 from six import text_type
-from ckan.common import config
 from ckan.model.core import State
 
 try:
@@ -146,7 +147,7 @@ def table_dict_save(table_dict, ModelClass, context):
     if not obj:
         obj = ModelClass()
 
-    for key, value in table_dict.iteritems():
+    for key, value in six.iteritems(table_dict):
         if isinstance(value, list):
             continue
         setattr(obj, key, value)
diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py
index 83e3c1a678f..e28827a36b1 100644
--- a/ckan/lib/dictization/model_dictize.py
+++ b/ckan/lib/dictization/model_dictize.py
@@ -11,8 +11,8 @@
 
 which builds the dictionary by iterating over the table columns.
 '''
-import datetime
 
+import six
 from six.moves.urllib.parse import urlsplit
 
 from ckan.common import config
@@ -76,7 +76,7 @@ def resource_list_dictize(res_list, context):
 
 def extras_dict_dictize(extras_dict, context):
     result_list = []
-    for name, extra in extras_dict.iteritems():
+    for name, extra in six.iteritems(extras_dict):
         dictized = d.table_dictize(extra, context)
         if not extra.state == 'active':
             continue
diff --git a/ckan/lib/dictization/model_save.py b/ckan/lib/dictization/model_save.py
index 8da4b059c1a..1bc3ea5e371 100644
--- a/ckan/lib/dictization/model_save.py
+++ b/ckan/lib/dictization/model_save.py
@@ -5,11 +5,10 @@
 import logging
 
 from sqlalchemy.orm import class_mapper
+import six
 from six import string_types
-from six.moves import map
 
 import ckan.lib.dictization as d
-import ckan.lib.helpers as h
 import ckan.authz as authz
 
 log = logging.getLogger(__name__)
@@ -40,7 +39,7 @@ def resource_dict_save(res_dict, context):
     # Resource extras not submitted will be removed from the existing extras
     # dict
     new_extras = {}
-    for key, value in res_dict.iteritems():
+    for key, value in six.iteritems(res_dict):
         if isinstance(value, list):
             continue
         if key in ('extras', 'revision_timestamp', 'tracking_summary'):
@@ -451,7 +450,7 @@ def package_api_to_dict(api1_dict, context):
 
     dictized = {}
 
-    for key, value in api1_dict.iteritems():
+    for key, value in six.iteritems(api1_dict):
         new_value = value
         if key == 'tags':
             if isinstance(value, string_types):
@@ -466,7 +465,7 @@ def package_api_to_dict(api1_dict, context):
 
             new_value = []
 
-            for extras_key, extras_value in updated_extras.iteritems():
+            for extras_key, extras_value in six.iteritems(updated_extras):
                 new_value.append({"key": extras_key,
                                   "value": extras_value})
 
@@ -490,7 +489,7 @@ def group_api_to_dict(api1_dict, context):
 
     dictized = {}
 
-    for key, value in api1_dict.iteritems():
+    for key, value in six.iteritems(api1_dict):
         new_value = value
         if key == 'packages':
             new_value = [{"id": item} for item in value]
@@ -601,7 +600,7 @@ def resource_view_dict_save(data_dict, context):
     if resource_view:
         data_dict['id'] = resource_view.id
     config = {}
-    for key, value in data_dict.iteritems():
+    for key, value in six.iteritems(data_dict):
         if key not in model.ResourceView.get_columns():
             config[key]  = value
     data_dict['config'] = config
diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py
index 395a6dbabba..f1ca57c46bd 100644
--- a/ckan/lib/helpers.py
+++ b/ckan/lib/helpers.py
@@ -944,7 +944,7 @@ def build_extra_admin_nav():
     admin_tabs_dict = config.get('ckan.admin_tabs')
     output = ''
     if admin_tabs_dict:
-        for k, v in admin_tabs_dict.iteritems():
+        for k, v in six.iteritems(admin_tabs_dict):
             if v['icon']:
                 output += build_nav_icon(k, v['label'], icon=v['icon'])
             else:
diff --git a/ckan/lib/i18n.py b/ckan/lib/i18n.py
index acc3a38f1fe..9236224c89d 100644
--- a/ckan/lib/i18n.py
+++ b/ckan/lib/i18n.py
@@ -380,7 +380,7 @@ def build_js_translations():
     # the POT files for that, since they contain all translation entries
     # (even those for which no translation exists, yet).
     js_entries = set()
-    for i18n_dir, domain in i18n_dirs.iteritems():
+    for i18n_dir, domain in six.iteritems(i18n_dirs):
         pot_file = os.path.join(i18n_dir, domain + u'.pot')
         if os.path.isfile(pot_file):
             js_entries.update(_get_js_translation_entries(pot_file))
@@ -395,7 +395,7 @@ def build_js_translations():
                     u'LC_MESSAGES',
                     domain + u'.po'
                 )
-                for i18n_dir, domain in i18n_dirs.iteritems()
+                for i18n_dir, domain in six.iteritems(i18n_dirs)
             ) if os.path.isfile(fn)
         ]
         if not po_files:
diff --git a/ckan/lib/lazyjson.py b/ckan/lib/lazyjson.py
index 54dd9242856..a91ecfccbf7 100644
--- a/ckan/lib/lazyjson.py
+++ b/ckan/lib/lazyjson.py
@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 from simplejson import loads, RawJSON, dumps
+import six
 from six import text_type
 
 
diff --git a/ckan/lib/navl/dictization_functions.py b/ckan/lib/navl/dictization_functions.py
index 3f2275b17e4..26846c09026 100644
--- a/ckan/lib/navl/dictization_functions.py
+++ b/ckan/lib/navl/dictization_functions.py
@@ -3,6 +3,7 @@
 import copy
 import json
 
+import six
 from six import text_type
 from ckan.common import config, _
 
@@ -105,7 +106,7 @@ def flatten_schema(schema, flattened=None, key=None):
     flattened = flattened or {}
     old_key = key or []
 
-    for key, value in schema.iteritems():
+    for key, value in six.iteritems(schema):
         new_key = old_key + [key]
         if isinstance(value, dict):
             flattened = flatten_schema(value, flattened, new_key)
@@ -152,7 +153,7 @@ def make_full_schema(data, schema):
         for key in combination[::2]:
             sub_schema = sub_schema[key]
 
-        for key, value in sub_schema.iteritems():
+        for key, value in six.iteritems(sub_schema):
             if isinstance(value, list):
                 full_schema[combination + (key,)] = value
 
@@ -378,7 +379,7 @@ def flatten_dict(data, flattened=None, old_key=None):
     flattened = flattened or {}
     old_key = old_key or []
 
-    for key, value in data.iteritems():
+    for key, value in six.iteritems(data):
         new_key = old_key + [key]
         if isinstance(value, list) and value and isinstance(value[0], dict):
             flattened = flatten_list(value, flattened, new_key)
diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py
index a88146b1cb9..18c5e8386cb 100644
--- a/ckan/lib/navl/validators.py
+++ b/ckan/lib/navl/validators.py
@@ -1,5 +1,6 @@
 # encoding: utf-8
 
+import six
 from six import text_type
 
 import ckan.lib.navl.dictization_functions as df
@@ -17,7 +18,7 @@ def identity_converter(key, data, errors, context):
 def keep_extras(key, data, errors, context):
 
     extras = data.pop(key, {})
-    for extras_key, value in extras.iteritems():
+    for extras_key, value in six.iteritems(extras):
         data[key[:-1] + (extras_key,)] = value
 
 def not_missing(key, data, errors, context):
diff --git a/ckan/lib/search/index.py b/ckan/lib/search/index.py
index 548f33a8b22..8238877bad8 100644
--- a/ckan/lib/search/index.py
+++ b/ckan/lib/search/index.py
@@ -221,7 +221,7 @@ def index_package(self, pkg_dict, defer_commit=False):
         for rel in subjects:
             type = rel['type']
             rel_dict[type].append(model.Package.get(rel['object_package_id']).name)
-        for key, value in rel_dict.iteritems():
+        for key, value in six.iteritems(rel_dict):
             if key not in pkg_dict:
                 pkg_dict[key] = value
 
diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py
index 1ac7a23f88a..004c11e6810 100644
--- a/ckan/logic/__init__.py
+++ b/ckan/logic/__init__.py
@@ -6,6 +6,7 @@
 from collections import defaultdict
 
 from werkzeug.local import LocalProxy
+import six
 from six import string_types, text_type
 
 import ckan.model as model
@@ -93,7 +94,7 @@ def prettify(field_name):
 
             summary = {}
 
-            for key, error in error_dict.iteritems():
+            for key, error in six.iteritems(error_dict):
                 if key == 'resources':
                     summary[_('Resources')] = _('Package resource(s) invalid')
                 elif key == 'extras':
@@ -194,7 +195,7 @@ def tuplize_dict(data_dict):
     May raise a DataError if the format of the key is incorrect.
     '''
     tuplized_dict = {}
-    for key, value in data_dict.iteritems():
+    for key, value in six.iteritems(data_dict):
         key_list = key.split('__')
         for num, key in enumerate(key_list):
             if num % 2 == 1:
@@ -209,7 +210,7 @@ def tuplize_dict(data_dict):
 def untuplize_dict(tuplized_dict):
 
     data_dict = {}
-    for key, value in tuplized_dict.iteritems():
+    for key, value in six.iteritems(tuplized_dict):
         new_key = '__'.join([str(item) for item in key])
         data_dict[new_key] = value
     return data_dict
@@ -428,7 +429,7 @@ def get_action(action):
                 # This needs to be resolved later
                 action_function.auth_audit_exempt = True
                 fetched_actions[name] = action_function
-    for name, func_list in chained_actions.iteritems():
+    for name, func_list in six.iteritems(chained_actions):
         if name not in fetched_actions and name not in _actions:
             # nothing to override from plugins or core
             raise NotFound('The action %r is not found for chained action' % (
diff --git a/ckan/logic/action/__init__.py b/ckan/logic/action/__init__.py
index f105c21001c..591f175ea89 100644
--- a/ckan/logic/action/__init__.py
+++ b/ckan/logic/action/__init__.py
@@ -3,6 +3,8 @@
 from copy import deepcopy
 import re
 
+import six
+
 from ckan.logic import NotFound
 from ckan.common import _
 
@@ -55,7 +57,7 @@ def prettify(field_name):
         return _(field_name.replace('_', ' '))
 
     summary = {}
-    for key, error in error_dict.iteritems():
+    for key, error in six.iteritems(error_dict):
         if key == 'resources':
             summary[_('Resources')] = _('Package resource(s) invalid')
         elif key == 'extras':
diff --git a/ckan/logic/action/update.py b/ckan/logic/action/update.py
index 5b9271a41d8..5dc1520310c 100644
--- a/ckan/logic/action/update.py
+++ b/ckan/logic/action/update.py
@@ -6,11 +6,10 @@
 import datetime
 import time
 import json
-import mimetypes
-import os
 
 from ckan.common import config
 import ckan.common as converters
+import six
 from six import text_type
 
 import ckan.lib.helpers as h
@@ -1186,7 +1185,7 @@ def config_option_update(context, data_dict):
         model.Session.rollback()
         raise ValidationError(errors)
 
-    for key, value in data.iteritems():
+    for key, value in six.iteritems(data):
 
         # Set full Logo url
         if key == 'ckan.site_logo' and value and not value.startswith('http')\
diff --git a/ckan/logic/converters.py b/ckan/logic/converters.py
index c87cad49ac8..3e26380792a 100644
--- a/ckan/logic/converters.py
+++ b/ckan/logic/converters.py
@@ -2,6 +2,7 @@
 
 import json
 
+import six
 from six import string_types, text_type
 
 import ckan.model as model
@@ -27,14 +28,14 @@ def convert_from_extras(key, data, errors, context):
 
     def remove_from_extras(data, key):
         to_remove = []
-        for data_key, data_value in data.iteritems():
+        for data_key, data_value in six.iteritems(data):
             if (data_key[0] == 'extras'
                 and data_key[1] == key):
                 to_remove.append(data_key)
         for item in to_remove:
             del data[item]
 
-    for data_key, data_value in data.iteritems():
+    for data_key, data_value in six.iteritems(data):
         if (data_key[0] == 'extras'
             and data_key[-1] == 'key'
             and data_value == key[-1]):
diff --git a/ckan/migration/revision_legacy_code.py b/ckan/migration/revision_legacy_code.py
index 543163e1e33..9f8a33e1986 100644
--- a/ckan/migration/revision_legacy_code.py
+++ b/ckan/migration/revision_legacy_code.py
@@ -330,7 +330,7 @@ class Revision(object):
     the revision attribute.
     '''
     def __init__(self, **kw):
-        for k, v in kw.iteritems():
+        for k, v in six.iteritems(kw):
             setattr(self, k, v)
 
     # @property
@@ -364,7 +364,7 @@ def create_object_version(mapper_fn, base_object, rev_table):
     # If not need to do an explicit check
     class MyClass(object):
         def __init__(self, **kw):
-            for k, v in kw.iteritems():
+            for k, v in six.iteritems(kw):
                 setattr(self, k, v)
 
     name = base_object.__name__ + u'Revision'
diff --git a/ckan/tests/controllers/test_template.py b/ckan/tests/controllers/test_template.py
index 9e5400e229e..380aac1a8fd 100644
--- a/ckan/tests/controllers/test_template.py
+++ b/ckan/tests/controllers/test_template.py
@@ -1,6 +1,7 @@
 # encoding: utf-8
 
 from nose.tools import assert_equal
+import six
 
 import ckan.tests.helpers as helpers
 
@@ -14,6 +15,6 @@ def test_content_type(self):
             u'/page.html': u'text/html; charset=utf-8',
         }
         app = self._get_test_app()
-        for url, expected in cases.iteritems():
+        for url, expected in six.iteritems(cases):
             response = app.get(url, status=200)
             assert_equal(response.headers.get(u'Content-Type'), expected)
diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py
index 1d2cc2ac4e2..1b90c221055 100644
--- a/ckan/tests/legacy/test_coding_standards.py
+++ b/ckan/tests/legacy/test_coding_standards.py
@@ -22,6 +22,7 @@
 import re
 import sys
 
+import six
 from six import StringIO
 
 import pycodestyle
@@ -672,7 +673,7 @@ def test_auths_have_action_fn_blacklist(self):
 
     def test_fn_signatures(self):
         errors = []
-        for name, fn in self.actions.iteritems():
+        for name, fn in six.iteritems(self.actions):
             args_info = inspect.getargspec(fn)
             if args_info.args != ['context', 'data_dict'] \
                     or args_info.varargs is not None \
@@ -685,7 +686,7 @@ def test_fn_signatures(self):
 
     def test_fn_docstrings(self):
         errors = []
-        for name, fn in self.actions.iteritems():
+        for name, fn in six.iteritems(self.actions):
             if not getattr(fn, '__doc__', None):
                 if name not in self.ACTION_NO_DOC_STR_BLACKLIST:
                     errors.append(name)
diff --git a/ckan/tests/test_common.py b/ckan/tests/test_common.py
index 47e0db49baa..42b3cfe6e7a 100644
--- a/ckan/tests/test_common.py
+++ b/ckan/tests/test_common.py
@@ -4,6 +4,8 @@
 import pylons
 
 from nose.tools import eq_, assert_not_equal as neq_, assert_raises
+
+import six
 from six import text_type
 
 from ckan.tests import helpers
@@ -95,7 +97,7 @@ def test_iteritems_works(self):
         my_conf[u'test_key_2'] = u'Test value 2'
 
         cnt = 0
-        for key, value in my_conf.iteritems():
+        for key, value in six.iteritems(my_conf):
             cnt += 1
             assert key.startswith(u'test_key_')
             assert value.startswith(u'Test value')
diff --git a/ckan/views/api.py b/ckan/views/api.py
index 94dd0762904..e74e6a82846 100644
--- a/ckan/views/api.py
+++ b/ckan/views/api.py
@@ -5,6 +5,7 @@
 import logging
 
 from flask import Blueprint, make_response
+import six
 from six import text_type
 from werkzeug.exceptions import BadRequest
 
@@ -162,7 +163,7 @@ def mixed(multi_dict):
            item or a string otherwise
         '''
         out = {}
-        for key, value in multi_dict.to_dict(flat=False).iteritems():
+        for key, value in six.iteritems(multi_dict.to_dict(flat=False)):
             out[key] = value[0] if len(value) == 1 else value
         return out
 
@@ -202,7 +203,7 @@ def mixed(multi_dict):
     if request.method == u'PUT' and not request_data:
         raise ValueError(u'Invalid request. Please use the POST method for '
                          'your request')
-    for field_name, file_ in request.files.iteritems():
+    for field_name, file_ in six.iteritems(request.files):
         request_data[field_name] = file_
     log.debug(u'Request data extracted: %r', request_data)
 
diff --git a/ckan/views/dataset.py b/ckan/views/dataset.py
index b52b8cb4557..ba7b85121d4 100644
--- a/ckan/views/dataset.py
+++ b/ckan/views/dataset.py
@@ -3,15 +3,15 @@
 from collections import OrderedDict
 from functools import partial
 from six.moves.urllib.parse import urlencode
-import datetime
 from datetime import datetime
 
-from flask import Blueprint, make_response
+from flask import Blueprint
 from flask.views import MethodView
 from ckan.common import asbool
+
+import six
 from six import string_types, text_type
 
-import ckan.lib.i18n as i18n
 import ckan.lib.base as base
 import ckan.lib.helpers as h
 import ckan.lib.navl.dictization_functions as dict_fns
@@ -24,7 +24,6 @@
 from ckan.lib.render import TemplateNotFound
 from ckan.lib.search import SearchError, SearchQueryError, SearchIndexError
 from ckan.views import LazyView
-from ckan.views.api import CONTENT_TYPES
 
 NotFound = logic.NotFound
 NotAuthorized = logic.NotAuthorized
@@ -343,7 +342,7 @@ def search(package_type):
     extra_vars[u'dataset_type'] = package_type
 
     # TODO: remove
-    for key, value in extra_vars.iteritems():
+    for key, value in six.iteritems(extra_vars):
         setattr(g, key, value)
 
     return base.render(
diff --git a/ckan/views/feed.py b/ckan/views/feed.py
index d6265e99df9..a12f5898275 100644
--- a/ckan/views/feed.py
+++ b/ckan/views/feed.py
@@ -4,6 +4,7 @@
 
 from six.moves.urllib.parse import urlparse
 from flask import Blueprint, make_response
+import six
 from six import text_type
 import webhelpers.feedgenerator
 from ckan.common import _, config, g, request, response
@@ -61,7 +62,7 @@ def _enclosure(pkg):
 
 def _set_extras(**kw):
     extras = []
-    for key, value in kw.iteritems():
+    for key, value in six.iteritems(kw):
         extras.append({key: value})
     return extras
 
@@ -361,7 +362,7 @@ def _feed_url(query, controller, action, **kwargs):
     Constructs the url for the given action.  Encoding the query
     parameters.
     """
-    for item in query.iteritems():
+    for item in six.iteritems(query):
         kwargs['query'] = item
     return h.url_for(controller=controller, action=action, **kwargs)
 
diff --git a/ckan/views/group.py b/ckan/views/group.py
index e744c1d7b67..eb5ef0d613f 100644
--- a/ckan/views/group.py
+++ b/ckan/views/group.py
@@ -6,6 +6,8 @@
 from six.moves.urllib.parse import urlencode
 
 from pylons.i18n import get_lang
+
+import six
 from six import string_types, text_type
 
 import ckan.lib.base as base
@@ -811,7 +813,7 @@ def post(self, id, group_type, is_organization, data=None):
         actions = form_names.intersection(actions_in_form)
         # ie7 puts all buttons in form params but puts submitted one twice
 
-        for key, value in request.form.to_dict().iteritems():
+        for key, value in six.iteritems(request.form.to_dict()):
             if value in [u'private', u'public']:
                 action = key.split(u'.')[-1]
                 break
diff --git a/ckan/views/resource.py b/ckan/views/resource.py
index a90da184c5a..0765e93c938 100644
--- a/ckan/views/resource.py
+++ b/ckan/views/resource.py
@@ -6,6 +6,7 @@
 import flask
 from flask.views import MethodView
 
+import six
 import ckan.lib.base as base
 import ckan.lib.datapreview as lib_datapreview
 import ckan.lib.helpers as h
@@ -194,7 +195,7 @@ def post(self, package_type, id):
 
         # see if we have any data that we are trying to save
         data_provided = False
-        for key, value in data.iteritems():
+        for key, value in six.iteritems(data):
             if (
                     (value or isinstance(value, cgi.FieldStorage))
                     and key != u'resource_type'):
diff --git a/ckanext/datastore/backend/postgres.py b/ckanext/datastore/backend/postgres.py
index e7f93894815..202fdce7d18 100644
--- a/ckanext/datastore/backend/postgres.py
+++ b/ckanext/datastore/backend/postgres.py
@@ -11,6 +11,7 @@
 import hashlib
 import json
 
+import six
 from six.moves.urllib.parse import (
     urlencode, unquote, urlunparse, parse_qsl, urlparse
 )
@@ -359,7 +360,7 @@ def _where_clauses(data_dict, fields_types):
     filters = data_dict.get('filters', {})
     clauses = []
 
-    for field, value in filters.iteritems():
+    for field, value in six.iteritems(filters):
         if field not in fields_types:
             continue
         field_array_type = _is_array_type(fields_types[field])
@@ -384,7 +385,7 @@ def _where_clauses(data_dict, fields_types):
             clauses.append((clause_str,))
         elif isinstance(q, dict):
             lang = _fts_lang(data_dict.get('lang'))
-            for field, value in q.iteritems():
+            for field, value in six.iteritems(q):
                 if field not in fields_types:
                     continue
                 query_field = _ts_query_alias(field)
@@ -425,7 +426,7 @@ def _textsearch_query(lang, q, plain):
         statements.append(query)
         rank_columns[u'rank'] = rank
     elif isinstance(q, dict):
-        for field, value in q.iteritems():
+        for field, value in six.iteritems(q):
             query, rank = _build_query_and_rank_statements(
                 lang, value, plain, field)
             statements.append(query)
@@ -1199,7 +1200,7 @@ def validate(context, data_dict):
     data_dict_copy.pop('records_format', None)
     data_dict_copy.pop('calculate_record_count', None)
 
-    for key, values in data_dict_copy.iteritems():
+    for key, values in six.iteritems(data_dict_copy):
         if not values:
             continue
         if isinstance(values, string_types):
diff --git a/ckanext/multilingual/plugin.py b/ckanext/multilingual/plugin.py
index 0976f633fd6..b284a682583 100644
--- a/ckanext/multilingual/plugin.py
+++ b/ckanext/multilingual/plugin.py
@@ -1,5 +1,6 @@
 # encoding: utf-8
 
+import six
 from six import string_types
 
 import ckan
@@ -225,7 +226,7 @@ def before_index(self, search_data):
 
         ## translate rest
         all_terms = []
-        for key, value in search_data.iteritems():
+        for key, value in six.iteritems(search_data):
             if key in KEYS_TO_IGNORE or key.startswith('title'):
                 continue
             if not isinstance(value, list):
@@ -247,7 +248,7 @@ def before_index(self, search_data):
             lang_field = 'text_' + translation['lang_code']
             text_field_items[lang_field].append(translation['term_translation'])
 
-        for key, value in text_field_items.iteritems():
+        for key, value in six.iteritems(text_field_items):
             search_data[key] = ' '.join(value)
 
         return search_data
diff --git a/ckanext/reclineview/plugin.py b/ckanext/reclineview/plugin.py
index a7436dd429d..625d858390c 100644
--- a/ckanext/reclineview/plugin.py
+++ b/ckanext/reclineview/plugin.py
@@ -2,6 +2,8 @@
 
 from logging import getLogger
 
+import six
+
 from ckan.common import json, config
 import ckan.plugins as p
 import ckan.plugins.toolkit as toolkit
@@ -18,7 +20,7 @@ def get_mapview_config():
     '''
     namespace = 'ckanext.spatial.common_map.'
     return {k.replace(namespace, ''): v
-            for k, v in config.iteritems()
+            for k, v in six.iteritems(config)
             if k.startswith(namespace)}
 
 
diff --git a/ckanext/textview/plugin.py b/ckanext/textview/plugin.py
index f2e3b7a5c59..2089975824b 100644
--- a/ckanext/textview/plugin.py
+++ b/ckanext/textview/plugin.py
@@ -2,6 +2,8 @@
 
 import logging
 
+import six
+
 from ckan.common import json
 import ckan.plugins as p
 import ckanext.resourceproxy.plugin as proxy
@@ -51,7 +53,7 @@ class TextView(p.SingletonPlugin):
     def update_config(self, config):
 
         formats = get_formats(config)
-        for key, value in formats.iteritems():
+        for key, value in six.iteritems(formats):
             setattr(self, key, value)
 
         self.no_jsonp_formats = (self.text_formats +

From f97addb57bf21a0040bca141e451f3abc510dcee Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Wed, 27 Nov 2019 21:09:09 +0100
Subject: [PATCH 20/30] [#4801] Update test to use Flask exception

---
 ckan/tests/config/test_middleware.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py
index a1b23378235..cdfe0ddf54e 100644
--- a/ckan/tests/config/test_middleware.py
+++ b/ckan/tests/config/test_middleware.py
@@ -650,7 +650,4 @@ def test_beaker_secret_is_used_by_default(self):
     @helpers.change_config('beaker.session.secret', None)
     def test_no_beaker_secret_crashes(self):
 
-        assert_raises(ValueError, helpers._get_test_app)
-
-        # TODO: When Pylons is finally removed, we should test for
-        # RuntimeError instead (thrown on `make_flask_stack`)
+        assert_raises(RuntimeError, helpers._get_test_app)

From 8500baea2867dcad771617ac84764d0ac990d9b7 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 28 Nov 2019 11:52:34 +0100
Subject: [PATCH 21/30] [#4801] Catch exception outside request context

---
 ckan/logic/__init__.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ckan/logic/__init__.py b/ckan/logic/__init__.py
index 004c11e6810..844efa1f63b 100644
--- a/ckan/logic/__init__.py
+++ b/ckan/logic/__init__.py
@@ -232,6 +232,9 @@ def _prepopulate_context(context):
     except AttributeError:
         # c.user not set
         pass
+    except RuntimeError:
+        # Outside of request context
+        pass
     except TypeError:
         # c not registered
         pass

From 9dff2edc47facd39808bb897fe9fda300c74418a Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 28 Nov 2019 11:56:33 +0100
Subject: [PATCH 22/30] [#4801] Rename configparser module

---
 ckan/lib/fanstatic_resources.py     | 4 ++--
 scripts/4042_fix_resource_extras.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/ckan/lib/fanstatic_resources.py b/ckan/lib/fanstatic_resources.py
index a0d4ad9e7a6..7700d0708b6 100644
--- a/ckan/lib/fanstatic_resources.py
+++ b/ckan/lib/fanstatic_resources.py
@@ -3,7 +3,7 @@
 import os.path
 import sys
 import logging
-import ConfigParser
+import configparser
 from ckan.common import config
 
 from fanstatic import Library, Resource, Group, get_library_registry
@@ -127,7 +127,7 @@ def create_resource(path, lib_name, count, inline=False, supersedes=None):
     # parse the resource.config file if it exists
     config_path = os.path.join(resource_path, 'resource.config')
     if os.path.exists(config_path):
-        config = ConfigParser.RawConfigParser()
+        config = configparser.RawConfigParser()
         config.read(config_path)
 
         if config.has_option('main', 'order'):
diff --git a/scripts/4042_fix_resource_extras.py b/scripts/4042_fix_resource_extras.py
index 7d4e0437ec1..113ac0896bd 100644
--- a/scripts/4042_fix_resource_extras.py
+++ b/scripts/4042_fix_resource_extras.py
@@ -28,7 +28,7 @@
 '''
 
 import json
-from ConfigParser import ConfigParser
+from configparser import ConfigParser
 from argparse import ArgumentParser
 from six.moves import input
 from sqlalchemy import create_engine

From 313f6515b2f704312126d9868afb63b54b9c0890 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Thu, 28 Nov 2019 12:16:42 +0100
Subject: [PATCH 23/30] [#4801] Upgrade fanstatic on to a py3 compatible
 version (only in py3)

---
 requirements.in  | 2 +-
 requirements.txt | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/requirements.in b/requirements.in
index ede48112b7f..4e75bc7dfd9 100644
--- a/requirements.in
+++ b/requirements.in
@@ -5,7 +5,7 @@ Babel==2.3.4
 Beaker==1.11.0
 bleach==3.0.2
 click==6.7
-fanstatic==0.12
+fanstatic==1.1
 Flask==1.1.1
 Flask-Babel==0.11.2
 Jinja2==2.10.1
diff --git a/requirements.txt b/requirements.txt
index a2c2365872a..64dba6abc52 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,11 +8,11 @@ alembic==1.0.0
 babel==2.3.4
 beaker==1.11.0
 bleach==3.0.2
-certifi==2019.9.11        # via requests
+certifi==2019.11.28       # via requests
 chardet==3.0.4            # via requests
 click==6.7
 decorator==4.4.1          # via sqlalchemy-migrate
-fanstatic==0.12
+fanstatic==1.1
 flask-babel==0.11.2
 flask==1.1.1
 funcsigs==1.0.2           # via beaker
@@ -27,7 +27,7 @@ passlib==1.6.5
 paste==1.7.5.1
 pastedeploy==2.0.1        # via pastescript
 pastescript==2.0.2
-pbr==5.4.3                # via sqlalchemy-migrate
+pbr==5.4.4                # via sqlalchemy-migrate
 polib==1.0.7
 psycopg2==2.8.2
 pysolr==3.6.0
@@ -43,6 +43,7 @@ repoze.who==2.3
 requests==2.22.0
 routes==1.13
 rq==1.0
+shutilwhich==1.1.0        # via fanstatic
 simplejson==3.10.0
 six==1.13.0               # via bleach, pastescript, python-dateutil, pyutilib, sqlalchemy-migrate
 sqlalchemy-migrate==0.12.0

From c1bfc70fb371190ea8c591992f92d96c761130f2 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Tue, 3 Dec 2019 23:12:08 +0100
Subject: [PATCH 24/30] [#4801] Fix configparser imports

---
 ckan/lib/fanstatic_resources.py     | 6 ++++--
 scripts/4042_fix_resource_extras.py | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/ckan/lib/fanstatic_resources.py b/ckan/lib/fanstatic_resources.py
index 7700d0708b6..2a8ac26fded 100644
--- a/ckan/lib/fanstatic_resources.py
+++ b/ckan/lib/fanstatic_resources.py
@@ -3,7 +3,9 @@
 import os.path
 import sys
 import logging
-import configparser
+
+from six.moves.configparser import RawConfigParser
+
 from ckan.common import config
 
 from fanstatic import Library, Resource, Group, get_library_registry
@@ -127,7 +129,7 @@ def create_resource(path, lib_name, count, inline=False, supersedes=None):
     # parse the resource.config file if it exists
     config_path = os.path.join(resource_path, 'resource.config')
     if os.path.exists(config_path):
-        config = configparser.RawConfigParser()
+        config = RawConfigParser()
         config.read(config_path)
 
         if config.has_option('main', 'order'):
diff --git a/scripts/4042_fix_resource_extras.py b/scripts/4042_fix_resource_extras.py
index 113ac0896bd..43a816538ef 100644
--- a/scripts/4042_fix_resource_extras.py
+++ b/scripts/4042_fix_resource_extras.py
@@ -28,7 +28,7 @@
 '''
 
 import json
-from configparser import ConfigParser
+from six.moves.configparser import ConfigParser
 from argparse import ArgumentParser
 from six.moves import input
 from sqlalchemy import create_engine

From 6e456b264e95867740fb97452eb6348c9ff8be39 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Wed, 4 Dec 2019 10:34:23 +0100
Subject: [PATCH 25/30] Fix bad merge

---
 ckan/plugins/toolkit.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ckan/plugins/toolkit.py b/ckan/plugins/toolkit.py
index a949202ef6c..a9c37b3e031 100644
--- a/ckan/plugins/toolkit.py
+++ b/ckan/plugins/toolkit.py
@@ -280,8 +280,6 @@ def _initialize(self):
         t['enqueue_job'] = enqueue_job
 
         if six.PY2:
-
-            t['literal'] = webhelpers.html.tags.literal
             t['response'] = pylons.response
             self.docstring_overrides['response'] = '''
 The Pylons response object.

From 733787ca0d63e22501ffe9f0f02e8d1fc37eb015 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 6 Dec 2019 16:37:12 +0100
Subject: [PATCH 26/30] [#4801] Remove test file added by mistake

---
 ckan/tests/migration/__init__.py              |  10 -
 .../migration/revision_legacy_code_tests.py   | 307 ------------------
 2 files changed, 317 deletions(-)
 delete mode 100644 ckan/tests/migration/__init__.py
 delete mode 100644 ckan/tests/migration/revision_legacy_code_tests.py

diff --git a/ckan/tests/migration/__init__.py b/ckan/tests/migration/__init__.py
deleted file mode 100644
index d94dac35161..00000000000
--- a/ckan/tests/migration/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-# encoding: utf-8
-
-'''**All migration scripts should have tests.**
-
-.. todo::
-
-   Write some tests for a migration script, and then use them as an example to
-   fill out this guidelines section.
-
-'''
diff --git a/ckan/tests/migration/revision_legacy_code_tests.py b/ckan/tests/migration/revision_legacy_code_tests.py
deleted file mode 100644
index 780d7c18439..00000000000
--- a/ckan/tests/migration/revision_legacy_code_tests.py
+++ /dev/null
@@ -1,307 +0,0 @@
-# encoding: utf-8
-
-# This file is at the top of the ckan repository, because it needs to be run
-# separately from all the other tests, because when it imports
-# revision_legacy_code.py it changes the core model, which causes a number of
-# test failures which we're not concerned about.
-
-from difflib import unified_diff
-from pprint import pprint, pformat
-
-from ckan import model
-
-import ckan.lib.search as search
-from ckan.lib.dictization.model_save import package_dict_save
-from ckan.lib.create_test_data import CreateTestData
-from ckan.tests import helpers
-
-from ckan.migration.revision_legacy_code import package_dictize_with_revisions as package_dictize
-from ckan.migration.revision_legacy_code import RevisionTableMappings, make_package_revision
-from ckan.migration.migrate_package_activity import PackageDictizeMonkeyPatch
-
-
-# tests here have been moved from ckan/tests/legacy/lib/test_dictization.py
-class TestPackageDictizeWithRevisions(object):
-    @classmethod
-    def setup_class(cls):
-
-        cls.package_expected = {
-            u'author': None,
-            u'author_email': None,
-            u'creator_user_id': None,
-            'extras': [
-                # extra_revision_table is no longer being populated because
-                # PackageExtra no longer has
-                # vdm.sqlalchemy.Revisioner(extra_revision_table) (removed in
-                # #4691) so don't test extras for the moment
-                # {'key': u'david', 'state': u'active', 'value': u'new_value'},
-                # {'key': u'genre', 'state': u'active', 'value': u'new_value'},
-                # {'key': u'original media', 'state': u'active',
-                #  'value': u'book'}
-                 ],
-            'groups': [{
-                        u'name': u'david',
-                        u'capacity': u'public',
-                        u'image_url': u'',
-                        u'image_display_url': u'',
-                        u'description': u'These are books that David likes.',
-                        u'display_name': u"Dave's books",
-                        u'type': u'group',
-                        u'state': u'active',
-                        u'is_organization': False,
-                        u'title': u"Dave's books",
-                        u"approval_status": u"approved"},
-                       {
-                        u'name': u'roger',
-                        u'capacity': u'public',
-                        u'description': u'Roger likes these books.',
-                        u'image_url': u'',
-                        'image_display_url': u'',
-                        'display_name': u"Roger's books",
-                        u'type': u'group',
-                        u'state': u'active',
-                        u'is_organization': False,
-                        u'title': u"Roger's books",
-                        u"approval_status": u"approved"}],
-            'isopen': True,
-            u'license_id': u'other-open',
-            'license_title': u'Other (Open)',
-            'organization': None,
-            u'owner_org': None,
-            u'maintainer': None,
-            u'maintainer_email': None,
-            u'name': u'annakarenina',
-            u'notes': u'Some test notes\n\n### A 3rd level heading\n\n**Some bolded text.**\n\n*Some italicized text.*\n\nForeign characters:\nu with umlaut \xfc\n66-style quote \u201c\nforeign word: th\xfcmb\n\nNeeds escaping:\nleft arrow <\n\n<http://ckan.net/>\n\n',
-            'num_resources': 2,
-            'num_tags': 3,
-            u'private': False,
-            'relationships_as_object': [],
-            'relationships_as_subject': [],
-            'resources': [{u'alt_url': u'alt123',
-                            u'cache_last_updated': None,
-                            u'cache_url': None,
-                            u'description': u'Full text. Needs escaping: " Umlaut: \xfc',
-                            u'format': u'plain text',
-                            u'hash': u'abc123',
-                            u'last_modified': None,
-                            u'mimetype': None,
-                            u'mimetype_inner': None,
-                            u'name': None,
-                            u'position': 0,
-                            u'resource_type': None,
-                            u'size': None,
-                            u'size_extra': u'123',
-                            u'url_type': None,
-                            u'state': u'active',
-                            u'url': u'http://datahub.io/download/x=1&y=2',},
-                           {u'alt_url': u'alt345',
-                            u'cache_last_updated': None,
-                            u'cache_url': None,
-                            u'description': u'Index of the novel',
-                            u'format': u'JSON',
-                            u'hash': u'def456',
-                            u'last_modified': None,
-                            u'mimetype': None,
-                            u'mimetype_inner': None,
-                            u'name': None,
-                            u'position': 1,
-                            u'resource_type': None,
-                            u'url_type': None,
-                            u'size': None,
-                            u'size_extra': u'345',
-                            u'state': u'active',
-                            u'url': u'http://datahub.io/index.json'}],
-            u'state': u'active',
-            'tags': [{u'name': u'Flexible \u30a1',
-                        'display_name': u'Flexible \u30a1',
-                        u'state': u'active'},
-                     {'display_name': u'russian',
-                         u'name': u'russian',
-                         u'state': u'active'},
-                     {'display_name': u'tolstoy',
-                         u'name': u'tolstoy',
-                         u'state': u'active'}],
-            u'title': u'A Novel By Tolstoy',
-            u'type': u'dataset',
-            u'url': u'http://datahub.io',
-            u'version': u'0.7a',
-        }
-
-    def setup(self):
-        helpers.reset_db()
-        search.clear_all()
-        CreateTestData.create()
-        make_package_revision(model.Package.by_name('annakarenina'))
-
-    def teardown(self):
-        helpers.reset_db()
-        search.clear_all()
-
-    def test_09_package_alter(self):
-
-        context = {"model": model,
-                   "session": model.Session,
-                   "user": 'testsysadmin'
-                   }
-
-        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina').one()
-
-        anna_dictized = package_dictize(anna1, context)
-
-        anna_dictized["name"] = u'annakarenina_changed'
-        anna_dictized["resources"][0]["url"] = u'http://new_url'
-
-        package_dict_save(anna_dictized, context)
-        model.Session.commit()
-        model.Session.remove()
-        make_package_revision(model.Package.by_name('annakarenina_changed'))
-
-        pkg = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one()
-
-        package_dictized = package_dictize(pkg, context)
-
-        resources_revisions = model.Session.query(RevisionTableMappings.instance().ResourceRevision).filter_by(package_id=anna1.id).all()
-
-        sorted_resource_revisions = sorted(resources_revisions, key=lambda x: (x.revision_timestamp, x.url))[::-1]
-        for res in sorted_resource_revisions:
-            print(res.id, res.revision_timestamp, res.state)
-        assert len(sorted_resource_revisions) == 4  # 2 resources originally, then make_package_revision saves them both again
-
-        # Make sure we remove changeable fields BEFORE we store the pretty-printed version
-        # for comparison
-        clean_package_dictized = self.remove_changable_columns(package_dictized)
-
-        anna_original = pformat(anna_dictized)
-        anna_after_save = pformat(clean_package_dictized)
-
-        assert self.remove_changable_columns(anna_dictized) == clean_package_dictized, \
-            "\n".join(unified_diff(anna_original.split("\n"), anna_after_save.split("\n")))
-
-        # changes to the package, relied upon by later tests
-        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed').one()
-        anna_dictized = package_dictize(anna1, context)
-        anna_dictized['name'] = u'annakarenina_changed2'
-        anna_dictized['resources'][0]['url'] = u'http://new_url2'
-        anna_dictized['tags'][0]['name'] = u'new_tag'
-        anna_dictized['tags'][0].pop('id')  # test if
-        anna_dictized['extras'][0]['value'] = u'new_value'
-
-        package_dict_save(anna_dictized, context)
-        model.Session.commit()
-        model.Session.remove()
-        make_package_revision(model.Package.by_name('annakarenina_changed2'))
-
-        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one()
-        anna_dictized = package_dictize(anna1, context)
-        anna_dictized['notes'] = 'wee'
-        anna_dictized['resources'].append({
-                            'format': u'plain text',
-                            'url': u'http://newurl'}
-                            )
-        anna_dictized['tags'].append({'name': u'newnew_tag'})
-        anna_dictized['extras'].append({'key': 'david',
-                                        'value': u'new_value'})
-
-        package_dict_save(anna_dictized, context)
-        model.Session.commit()
-        model.Session.remove()
-        make_package_revision(model.Package.by_name('annakarenina_changed2'))
-
-        context = {'model': model,
-                   'session': model.Session}
-
-        anna1 = model.Session.query(model.Package).filter_by(name='annakarenina_changed2').one()
-
-        pkgrevisions = model.Session.query(RevisionTableMappings.instance().PackageRevision).filter_by(id=anna1.id).all()
-        sorted_packages = sorted(pkgrevisions, key=lambda x: x.revision_timestamp)
-
-        context['revision_id'] = sorted_packages[0].revision_id  # original state
-
-        with PackageDictizeMonkeyPatch():
-            first_dictized = self.remove_changable_columns(package_dictize(anna1, context))
-            assert self.remove_changable_columns(self.package_expected) == first_dictized
-
-            context['revision_id'] = sorted_packages[1].revision_id
-
-            second_dictized = self.remove_changable_columns(package_dictize(anna1, context))
-
-            first_dictized["name"] = u'annakarenina_changed'
-            first_dictized["resources"][0]["url"] = u'http://new_url'
-
-            assert second_dictized == first_dictized
-
-            context['revision_id'] = sorted_packages[2].revision_id
-            third_dictized = self.remove_changable_columns(package_dictize(anna1, context))
-
-            second_dictized['name'] = u'annakarenina_changed2'
-            second_dictized['resources'][0]['url'] = u'http://new_url2'
-            second_dictized['tags'][0]['name'] = u'new_tag'
-            second_dictized['tags'][0]['display_name'] = u'new_tag'
-            second_dictized['state'] = 'active'
-
-            print('\n'.join(unified_diff(pformat(second_dictized).split('\n'), pformat(third_dictized).split('\n'))))
-            assert second_dictized == third_dictized
-
-            context['revision_id'] = sorted_packages[3].revision_id  # original state
-            forth_dictized = self.remove_changable_columns(package_dictize(anna1, context))
-
-        third_dictized['notes'] = 'wee'
-        third_dictized['resources'].insert(2, {
-            u'cache_last_updated': None,
-            u'cache_url': None,
-            u'description': u'',
-            u'format': u'plain text',
-            u'hash': u'',
-            u'last_modified': None,
-            u'mimetype': None,
-            u'mimetype_inner': None,
-            u'name': None,
-            u'position': 2,
-            u'resource_type': None,
-            u'url_type': None,
-            u'size': None,
-            u'state': u'active',
-            u'url': u'http://newurl'})
-        third_dictized['num_resources'] = third_dictized['num_resources'] + 1
-
-        third_dictized['tags'].insert(1, {'name': u'newnew_tag', 'display_name': u'newnew_tag', 'state': 'active'})
-        third_dictized['num_tags'] = third_dictized['num_tags'] + 1
-        third_dictized['state'] = 'active'
-        third_dictized['state'] = 'active'
-
-        pprint(third_dictized)
-        pprint(forth_dictized)
-
-        assert third_dictized == forth_dictized
-
-    def remove_changable_columns(self, dict, remove_package_id=False):
-        ids_to_keep = ['license_id', 'creator_user_id']
-        if not remove_package_id:
-            ids_to_keep.append('package_id')
-
-        for key, value in dict.items():
-            if key.endswith('id') and key not in ids_to_keep:
-                dict.pop(key)
-            if key == 'created':
-                dict.pop(key)
-            if 'timestamp' in key:
-                dict.pop(key)
-            if key in ['metadata_created','metadata_modified']:
-                dict.pop(key)
-            if isinstance(value, list):
-                for new_dict in value:
-                    self.remove_changable_columns(new_dict,
-                        key in ['resources', 'extras'] or remove_package_id)
-
-            # TEMPORARY HACK - we remove 'extras' so they aren't tested. This
-            # is due to package_extra_revisions being migrated from ckan/model
-            # in #4691 but not the rest of the model revisions just yet. Until
-            # we finish this work (#4664) it is hard to get this working -
-            # extra_revision_table is no longer being populated because
-            # PackageExtra no longer has
-            # vdm.sqlalchemy.Revisioner(extra_revision_table). However #4664
-            # will allow use to manually create revisions and test this again.
-            if key == 'extras':
-                dict.pop(key)
-            # END OF HACK
-        return dict

From 8d09297e4f8bfa841ae18af817b037ff979d98d2 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Fri, 6 Dec 2019 17:00:45 +0100
Subject: [PATCH 27/30] Fix sphinx warning

---
 doc/contributing/testing.rst | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/doc/contributing/testing.rst b/doc/contributing/testing.rst
index d6918af7339..59b6e171fc5 100644
--- a/doc/contributing/testing.rst
+++ b/doc/contributing/testing.rst
@@ -454,12 +454,6 @@ Writing :mod:`ckan.plugins` tests
 .. automodule:: ckan.tests.plugins
 
 
-Writing :mod:`ckan.migration` tests
------------------------------------
-
-.. automodule:: ckan.tests.migration
-
-
 Writing :mod:`ckan.ckanext` tests
 ---------------------------------
 

From fde7aee8ad69ef0706cf854d523b1f7eaab8ada7 Mon Sep 17 00:00:00 2001
From: amercader <amercadero@gmail.com>
Date: Sat, 7 Dec 2019 00:08:45 +0100
Subject: [PATCH 28/30] Remove files from coding standards tests

---
 ckan/tests/legacy/test_coding_standards.py | 1 -
 ckan/tests/test_coding_standards.py        | 1 -
 2 files changed, 2 deletions(-)

diff --git a/ckan/tests/legacy/test_coding_standards.py b/ckan/tests/legacy/test_coding_standards.py
index eea857a7e10..60704e8e2fd 100644
--- a/ckan/tests/legacy/test_coding_standards.py
+++ b/ckan/tests/legacy/test_coding_standards.py
@@ -456,7 +456,6 @@ class TestPep8(object):
         'ckan/tests/legacy/schema/test_schema.py',
         'ckan/tests/legacy/test_plugins.py',
         'ckan/tests/legacy/test_versions.py',
-        'ckan/tests/migration/revision_legacy_code_tests.py',
         'test_revision_legacy_code.py',
         'ckan/websetup.py',
         'ckanext/datastore/bin/datastore_setup.py',
diff --git a/ckan/tests/test_coding_standards.py b/ckan/tests/test_coding_standards.py
index 4124bdf33c0..5256ef829df 100644
--- a/ckan/tests/test_coding_standards.py
+++ b/ckan/tests/test_coding_standards.py
@@ -562,7 +562,6 @@ def find_unprefixed_string_literals(filename):
     u'ckan/tests/logic/test_schema.py',
     u'ckan/tests/logic/test_validators.py',
     u'ckan/tests/migration/__init__.py',
-    u'ckan/tests/migration/revision_legacy_code_tests.py',
     u'test_revision_legacy_code.py',
     u'ckan/tests/model/__init__.py',
     u'ckan/tests/model/test_license.py',

From 6c51a519735ee713f64fd91c9dc48050f744699d Mon Sep 17 00:00:00 2001
From: Jari Voutilainen <jari.voutilainen@iki.fi>
Date: Sat, 7 Dec 2019 17:18:34 +0200
Subject: [PATCH 29/30] [#5099] Fix broken translation in image view
 placeholder

---
 ckanext/imageview/theme/templates/image_form.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ckanext/imageview/theme/templates/image_form.html b/ckanext/imageview/theme/templates/image_form.html
index 5ac719d6b10..bea9b334be2 100644
--- a/ckanext/imageview/theme/templates/image_form.html
+++ b/ckanext/imageview/theme/templates/image_form.html
@@ -1,3 +1,3 @@
 {% import 'macros/form.html' as form %}
 
-{{ form.input('image_url', id='field-image_url', label=_('Image url'), placeholder=_('eg. http://example.com/image.jpg  (if blank uses resource url)'), value=data.image_url, error=errors.image_url, classes=['control-full', 'control-large']) }}
+{{ form.input('image_url', id='field-image_url', label=_('Image url'), placeholder=_('eg. http://example.com/image.jpg (if blank uses resource url)'), value=data.image_url, error=errors.image_url, classes=['control-full', 'control-large']) }}

From a84859efcbfdd335b42e8f80c21f96a5ec045b3d Mon Sep 17 00:00:00 2001
From: howff <3064316+howff@users.noreply.github.com>
Date: Mon, 9 Dec 2019 10:17:41 +0000
Subject: [PATCH 30/30] Change arrow notation to lambda function

so that it's compatible with Internet Explorer
---
 ckan/public/base/javascript/client.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/ckan/public/base/javascript/client.js b/ckan/public/base/javascript/client.js
index b7fe576ebd0..b016eb3ccac 100644
--- a/ckan/public/base/javascript/client.js
+++ b/ckan/public/base/javascript/client.js
@@ -161,7 +161,10 @@
 
       var map = {};
       // If given a 'result' array then convert it into a Result dict inside a Result dict.
-      data = data.result ? { 'ResultSet': { 'Result': data.result.map(x => ({'Name': x})) } } : data;
+      // new syntax (not used until all browsers support arrow notation):
+      //data = data.result ? { 'ResultSet': { 'Result': data.result.map(x => ({'Name': x})) } } : data;
+      // compatible syntax:
+      data = data.result ? { 'ResultSet': { 'Result': data.result.map(function(val){ return { 'Name' :val } }) } } : data;
       // If given a Result dict inside a ResultSet dict then use the Result dict.
       var raw = jQuery.isArray(data) ? data : data.ResultSet && data.ResultSet.Result || {};