From cf1575e749ea88030f6613764ac2018153fe170c Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Wed, 25 May 2016 15:45:18 +0100 Subject: [PATCH 01/17] Removes reference to old session code [ref #3053] --- ckan/views/api.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/ckan/views/api.py b/ckan/views/api.py index e094a0346ee..cdeb588a081 100644 --- a/ckan/views/api.py +++ b/ckan/views/api.py @@ -100,17 +100,14 @@ def _identify_user_default(): g.user = g.user.decode('utf8') g.userobj = model.User.by_name(g.user) if g.userobj is None or not g.userobj.is_active(): - # This occurs when a user that was still logged in is deleted, - # or when you are logged in, clean db - # and then restart (or when you change your username) - # There is no user object, so even though repoze thinks you - # are logged in and your cookie has ckan_display_name, we - # need to force user to logout and login again to get the - # User object. - - # TODO: this should not be done here - # session['lang'] = request.environ.get('CKAN_LANG') - # session.save() + + # This occurs when a user that was still logged in is deleted, or + # when you are logged in, clean db and then restart (or when you + # change your username). There is no user object, so even though + # repoze thinks you are logged in and your cookie has + # ckan_display_name, we need to force user to logout and login + # again to get the User object. + ev = request.environ if 'repoze.who.plugins' in ev: pth = getattr(ev['repoze.who.plugins']['friendlyform'], From 5ffa7b4d5e8cc11b48f550675f539ef22802c766 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 26 May 2016 11:00:13 +0100 Subject: [PATCH 02/17] Get app debug status from app_config. Debug status defined in the ini file for Flask app. This helps in tests where we don't want debug to be hardcoded when the flask stack is made. --- ckan/config/middleware.py | 10 +++++++--- ckanext/example_flask_iroutes/tests/test_routes.py | 5 ----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index a21d75f5dba..754457c31a6 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -245,11 +245,16 @@ def __getattr__(self, name): def make_flask_stack(conf, **app_conf): - """ This has to pass the flask app through all the same middleware that - Pylons used """ + """ + This passes the flask app through most of the same middleware that Pylons + uses. + """ + + debug = app_conf.get('debug', True) root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) app = CKANFlask(__name__) + app.debug = debug app.template_folder = os.path.join(root, 'templates') app.app_ctx_globals_class = CKAN_AppCtxGlobals @@ -257,7 +262,6 @@ def make_flask_stack(conf, **app_conf): # secret key needed for flask-debug-toolbar app.config['SECRET_KEY'] = '' - app.debug = True app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False DebugToolbarExtension(app) diff --git a/ckanext/example_flask_iroutes/tests/test_routes.py b/ckanext/example_flask_iroutes/tests/test_routes.py index fa8110c76c8..e1feab9742f 100644 --- a/ckanext/example_flask_iroutes/tests/test_routes.py +++ b/ckanext/example_flask_iroutes/tests/test_routes.py @@ -37,11 +37,6 @@ def setup(self): self.app = helpers._get_test_app() flask_app = self._find_flask_app(self.app) - # Blueprints can't be registered after the app has been setup. For - # some reason, if debug is True, the app will have exited its initial - # state, and can't have new registrations. Set debug=False to ensure - # we can continue to register blueprints. - flask_app.debug = False # Install plugin and register its blueprint if not plugins.plugin_loaded('example_flask_iroutes'): plugins.load('example_flask_iroutes') From f7f27659ba4d79c4b92355c7ecad7b0dc4aa19e6 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 26 May 2016 11:13:27 +0100 Subject: [PATCH 03/17] Move `find_flask_app` helper to middleware.py This useful function can find the flask app when passed a wsgi stack. --- ckan/config/middleware.py | 28 +++++++++++++++++ .../tests/test_routes.py | 30 ++----------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 754457c31a6..d3012d935f9 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -744,3 +744,31 @@ def cleanup_pylons_response_string(environ): environ['pylons.controller']._py_object.response._body = msg except (KeyError, AttributeError): pass + + +def find_flask_app(test_app): + ''' + Helper function to recursively search the wsgi stack in `test_app` until + the flask_app is discovered. + + Relies on each layer of the stack having a reference to the app they + wrap in either a .app attribute or .apps list. + ''' + if isinstance(test_app, CKANFlask): + return test_app + + try: + app = test_app.apps['flask_app'].app + except (AttributeError, KeyError): + pass + else: + return find_flask_app(app) + + try: + app = test_app.app + except AttributeError: + print('No .app attribute. ' + 'Have all layers of the stack got ' + 'a reference to the app they wrap?') + else: + return find_flask_app(app) diff --git a/ckanext/example_flask_iroutes/tests/test_routes.py b/ckanext/example_flask_iroutes/tests/test_routes.py index e1feab9742f..27c33b79219 100644 --- a/ckanext/example_flask_iroutes/tests/test_routes.py +++ b/ckanext/example_flask_iroutes/tests/test_routes.py @@ -1,41 +1,15 @@ from nose.tools import eq_, ok_ -from ckan.config.middleware import CKANFlask +from ckan.config.middleware import find_flask_app import ckan.plugins as plugins import ckan.tests.helpers as helpers class TestFlaskIRoutes(helpers.FunctionalTestBase): - @classmethod - def _find_flask_app(cls, test_app): - '''Recursively search the wsgi stack until the flask_app is - discovered. - - Relies on each layer of the stack having a reference to the app they - wrap in either a .app attribute or .apps list. - ''' - if isinstance(test_app, CKANFlask): - return test_app - - try: - app = test_app.apps['flask_app'].app - except (AttributeError, KeyError): - pass - else: - return cls._find_flask_app(app) - - try: - app = test_app.app - except AttributeError: - print('No .app attribute. ' - 'Have all layers of the stack got ' - 'a reference to the app they wrap?') - else: - return cls._find_flask_app(app) def setup(self): self.app = helpers._get_test_app() - flask_app = self._find_flask_app(self.app) + flask_app = find_flask_app(self.app) # Install plugin and register its blueprint if not plugins.plugin_loaded('example_flask_iroutes'): From d4367d43a0ac54677eb2493e150342000bff54ee Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 26 May 2016 13:25:12 +0100 Subject: [PATCH 04/17] Use Beaker for Flask session interface --- ckan/common.py | 25 ++++++++++++++++++++++++- ckan/config/middleware.py | 13 +++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/ckan/common.py b/ckan/common.py index eed8537c1cd..0aafa71afae 100644 --- a/ckan/common.py +++ b/ckan/common.py @@ -12,7 +12,7 @@ from flask.ext.babel import gettext as flask_gettext from pylons.i18n import _ as pylons_gettext, ungettext -from pylons import g, session, response +from pylons import g, response import simplejson as json try: @@ -87,3 +87,26 @@ def __delattr__(self, name): c = PylonsStyleContext() + + +class Session(): + + def __getattr__(self, name): + if is_flask(): + return getattr(flask.session, name, None) + else: + return getattr(pylons.session, name, None) + + def __setattr__(self, name, value): + if is_flask(): + return setattr(flask.session, name, value) + else: + return setattr(pylons.session, name, value) + + def __delattr__(self, name): + if is_flask(): + return delattr(flask.session, name, None) + else: + return delattr(pylons.session, name, None) + +session = Session() diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index d3012d935f9..19ca8311ec3 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -28,6 +28,7 @@ from flask import request as flask_request from flask import _request_ctx_stack from flask.ctx import _AppCtxGlobals +from flask.sessions import SessionInterface from werkzeug.exceptions import HTTPException from werkzeug.test import create_environ, run_wsgi_app from flask.ext.babel import Babel @@ -260,6 +261,18 @@ def make_flask_stack(conf, **app_conf): # Do all the Flask-specific stuff before adding other middlewares + # Use Beaker as the Flask session interface + class BeakerSessionInterface(SessionInterface): + def open_session(self, app, request): + session = request.environ['beaker.session'] + return session + + def save_session(self, app, session, response): + session.save() + + app.wsgi_app = SessionMiddleware(app.wsgi_app) + app.session_interface = BeakerSessionInterface() + # secret key needed for flask-debug-toolbar app.config['SECRET_KEY'] = '' app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False From ac4ee7622de35fc296ae0280dc15725343bd58df Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 26 May 2016 13:32:44 +0100 Subject: [PATCH 05/17] Helper moved to test helpers --- ckan/config/middleware.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 19ca8311ec3..21d3fca5a2a 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -757,31 +757,3 @@ def cleanup_pylons_response_string(environ): environ['pylons.controller']._py_object.response._body = msg except (KeyError, AttributeError): pass - - -def find_flask_app(test_app): - ''' - Helper function to recursively search the wsgi stack in `test_app` until - the flask_app is discovered. - - Relies on each layer of the stack having a reference to the app they - wrap in either a .app attribute or .apps list. - ''' - if isinstance(test_app, CKANFlask): - return test_app - - try: - app = test_app.apps['flask_app'].app - except (AttributeError, KeyError): - pass - else: - return find_flask_app(app) - - try: - app = test_app.app - except AttributeError: - print('No .app attribute. ' - 'Have all layers of the stack got ' - 'a reference to the app they wrap?') - else: - return find_flask_app(app) From 28bc9c4a340a040a31f3120f4c9e73f83ace6719 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 26 May 2016 17:16:57 +0100 Subject: [PATCH 06/17] Proxy beaker session for universal support. Both Flask and Pylons can share the same beaker session objects, with a session proxy in common.py. This commit includes tests for cross app session access: - flask view populates flash message in session, rendered by flask view - flask view populates flash message in session, rendered by pylons action - pylons action populates flash message in session, rendered by flask view --- ckan/config/middleware.py | 21 ++- ckan/templates/base.html | 2 +- ckan/templates/tests/flash_messages.html | 14 ++ ckan/tests/config/test_sessions.py | 126 ++++++++++++++++++ ckanext/example_flask_iroutes/plugin.py | 10 ++ .../templates/about_base.html | 5 + setup.py | 1 + 7 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 ckan/templates/tests/flash_messages.html create mode 100644 ckan/tests/config/test_sessions.py create mode 100644 ckanext/example_flask_iroutes/templates/about_base.html diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index 21d3fca5a2a..ffa6bb14230 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -261,6 +261,11 @@ def make_flask_stack(conf, **app_conf): # Do all the Flask-specific stuff before adding other middlewares + # secret key needed for flask-debug-toolbar + app.config['SECRET_KEY'] = '' + app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False + DebugToolbarExtension(app) + # Use Beaker as the Flask session interface class BeakerSessionInterface(SessionInterface): def open_session(self, app, request): @@ -270,14 +275,18 @@ def open_session(self, app, request): def save_session(self, app, session, response): session.save() - app.wsgi_app = SessionMiddleware(app.wsgi_app) + cache_dir = app_conf.get('cache_dir') or app_conf.get('cache.dir') + session_opts = { + 'session.data_dir': '{data_dir}/sessions'.format( + data_dir=cache_dir), + 'session.key': app_conf.get('beaker.session.key'), + 'session.cookie_expires': + app_conf.get('beaker.session.cookie_expires'), + 'session.secret': app_conf.get('beaker.session.secret') + } + app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts) app.session_interface = BeakerSessionInterface() - # secret key needed for flask-debug-toolbar - app.config['SECRET_KEY'] = '' - app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False - DebugToolbarExtension(app) - # Add jinja2 extensions and filters extensions = [ 'jinja2.ext.do', 'jinja2.ext.with_', diff --git a/ckan/templates/base.html b/ckan/templates/base.html index 711475b1f7f..e1ac527bf86 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -86,7 +86,7 @@ {# Allows custom attributes to be added to the tag #} - + {# The page block allows you to add content to the page. Most of the time it is diff --git a/ckan/templates/tests/flash_messages.html b/ckan/templates/tests/flash_messages.html new file mode 100644 index 00000000000..f91e1d85d49 --- /dev/null +++ b/ckan/templates/tests/flash_messages.html @@ -0,0 +1,14 @@ + + + + Hello + + + Flash messages: + {% for message in h.flash.pop_messages() | list %} +
+ {{ message.category }}: {{ h.literal(message) }} +
+ {% endfor %} + + diff --git a/ckan/tests/config/test_sessions.py b/ckan/tests/config/test_sessions.py new file mode 100644 index 00000000000..21ff1de85a4 --- /dev/null +++ b/ckan/tests/config/test_sessions.py @@ -0,0 +1,126 @@ +from nose.tools import ok_ + +from flask import Blueprint +from flask import render_template +from flask import redirect as flask_redirect +from flask import url_for +from ckan.lib.base import redirect as pylons_redirect +from ckan.lib.base import render as pylons_render + +import ckan.plugins as p +import ckan.tests.helpers as helpers +import ckan.lib.helpers as h + + +class TestCrossFlaskPylonsFlashMessages(helpers.FunctionalTestBase): + ''' + Test that flash message set in the Pylons controller can be accessed by + Flask views, and visa versa. + ''' + + def setup(self): + self.app = helpers._get_test_app() + self.flask_app = helpers.find_flask_app(self.app) + + # Install plugin and register its blueprint + if not p.plugin_loaded('test_flash_plugin'): + p.load('test_flash_plugin') + plugin = p.get_plugin('test_flash_plugin') + self.flask_app.register_blueprint(plugin.get_blueprint(), + prioritise_rules=True) + + def test_flash_populated_by_flask_redirect_to_flask(self): + ''' + Flash store is populated by Flask view is accessible by another Flask + view. + ''' + res = self.app.get( + '/flask_add_flash_message_redirect_to_flask').follow() + + ok_("This is a success message populate by Flask" in res.body) + + def test_flash_populated_in_pylons_action_redirect_to_flask(self): + ''' + Flash store is populated by pylons action is accessible by Flask view. + ''' + res = self.app.get('/pylons_add_flash_message_redirect_view').follow() + + ok_("This is a success message populate by Pylons" in res.body) + + def test_flash_populated_in_flask_view_redirect_to_pylons(self): + ''' + Flash store is populated by flask view is accessible by pylons action. + ''' + res = self.app.get('/flask_add_flash_message_redirect_pylons').follow() + + ok_("This is a success message populate by Flask" in res.body) + + +class FlashMessagePlugin(p.SingletonPlugin): + ''' + A Flask and Pylons compatible IRoutes plugin to add Flask views and Pylons + actions to display flash messages. + ''' + + p.implements(p.IRoutes, inherit=True) + + def flash_message_view(self): + '''Flask view that renders the flash message html template.''' + return render_template('tests/flash_messages.html') + + def add_flash_message_view_redirect_to_flask(self): + '''Add flash message, then redirect to Flask view to render it.''' + h.flash_success("This is a success message populate by Flask") + return flask_redirect(url_for('test_flash_plugin.flash_message_view')) + + def add_flash_message_view_redirect_to_pylons(self): + '''Add flash message, then redirect to view that renders it''' + h.flash_success("This is a success message populate by Flask") + return flask_redirect('/pylons_view_flash_message') + + def get_blueprint(self): + '''Return Flask Blueprint object to be registered by the Flask app.''' + + # Create Blueprint for plugin + blueprint = Blueprint(self.name, self.__module__) + blueprint.template_folder = 'templates' + # Add plugin url rules to Blueprint object + rules = [ + ('/flask_add_flash_message_redirect_to_flask', 'add_flash_message', + self.add_flash_message_view_redirect_to_flask), + ('/flask_add_flash_message_redirect_pylons', + 'add_flash_message_view_redirect_to_pylons', + self.add_flash_message_view_redirect_to_pylons), + ('/flask_view_flash_message', 'flash_message_view', + self.flash_message_view), + ] + for rule in rules: + blueprint.add_url_rule(*rule) + + return blueprint + + controller = \ + 'ckan.tests.config.test_sessions:PylonsAddFlashMessageController' + + def before_map(self, _map): + '''Update the pylons route map to be used by the Pylons app.''' + _map.connect('/pylons_add_flash_message_redirect_view', + controller=self.controller, + action='add_flash_message_redirect') + + _map.connect('/pylons_view_flash_message', + controller=self.controller, + action='flash_message_action') + return _map + + +class PylonsAddFlashMessageController(p.toolkit.BaseController): + + def flash_message_action(self): + '''Pylons view to render flash messages in a template.''' + return pylons_render('tests/flash_messages.html') + + def add_flash_message_redirect(self): + # Adds a flash message and redirects to flask view + h.flash_success('This is a success message populate by Pylons') + return pylons_redirect('/flask_view_flash_message') diff --git a/ckanext/example_flask_iroutes/plugin.py b/ckanext/example_flask_iroutes/plugin.py index 8312add5586..2edc7fbfe06 100644 --- a/ckanext/example_flask_iroutes/plugin.py +++ b/ckanext/example_flask_iroutes/plugin.py @@ -14,6 +14,14 @@ def override_pylons_about(): return render_template('about.html') +def override_pylons_about_with_core_template(): + ''' + Override the pylons about controller to render the core about page + template. + ''' + return render_template('home/about.html') + + def override_flask_hello(): '''A simple replacement for the flash Hello view function.''' html = ''' @@ -75,6 +83,8 @@ def get_blueprint(self): rules = [ ('/hello_plugin', 'hello_plugin', hello_plugin), ('/about', 'about', override_pylons_about), + ('/about_core', 'about_core', + override_pylons_about_with_core_template), ('/hello', 'hello', override_flask_hello), ('/helper_not_here', 'helper_not_here', helper_not_here), ('/helper', 'helper_here', helper_here), diff --git a/ckanext/example_flask_iroutes/templates/about_base.html b/ckanext/example_flask_iroutes/templates/about_base.html new file mode 100644 index 00000000000..b561090bd5a --- /dev/null +++ b/ckanext/example_flask_iroutes/templates/about_base.html @@ -0,0 +1,5 @@ +{% extends "page.html" %} + +{% block page %} + This about page extends from page.html +{% endblock page %} diff --git a/setup.py b/setup.py index 5ec8f288038..0d01eaa571f 100644 --- a/setup.py +++ b/setup.py @@ -160,6 +160,7 @@ 'test_datastore_view = ckan.tests.lib.test_datapreview:MockDatastoreBasedResourceView', 'test_datapusher_plugin = ckanext.datapusher.tests.test_interfaces:FakeDataPusherPlugin', 'test_routing_plugin = ckan.tests.config.test_middleware:MockRoutingPlugin', + 'test_flash_plugin = ckan.tests.config.test_sessions:FlashMessagePlugin', 'test_helpers_plugin = ckan.tests.lib.test_helpers:TestHelpersPlugin', ], 'babel.extractors': [ From 973d8b0cb1067da790bde7c203f5bdaffe40f62c Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 26 May 2016 17:21:56 +0100 Subject: [PATCH 07/17] Revert removal of body attributes. Mistakenly removed for testing. --- ckan/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/templates/base.html b/ckan/templates/base.html index e1ac527bf86..711475b1f7f 100644 --- a/ckan/templates/base.html +++ b/ckan/templates/base.html @@ -86,7 +86,7 @@ {# Allows custom attributes to be added to the tag #} - + {# The page block allows you to add content to the page. Most of the time it is From fcd62c1b0db04004ac2c963db0e23c94b26cd963 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Mon, 30 May 2016 14:37:52 +0100 Subject: [PATCH 08/17] Fix extension list in setup.py Missing common after list item. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e1b6ea1d453..1ab6bd245f4 100644 --- a/setup.py +++ b/setup.py @@ -138,7 +138,7 @@ 'example_itranslation = ckanext.example_itranslation.plugin:ExampleITranslationPlugin', 'example_iconfigurer_v1 = ckanext.example_iconfigurer.plugin_v1:ExampleIConfigurerPlugin', 'example_iconfigurer_v2 = ckanext.example_iconfigurer.plugin_v2:ExampleIConfigurerPlugin', - 'example_flask_iroutes = ckanext.example_flask_iroutes.plugin:ExampleFlaskIRoutesPlugin' + 'example_flask_iroutes = ckanext.example_flask_iroutes.plugin:ExampleFlaskIRoutesPlugin', 'example_iuploader = ckanext.example_iuploader.plugin:ExampleIUploader', ], 'ckan.system_plugins': [ From 9fed8b483bf2e30b1307df4080d61ca5c24fed65 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Mon, 30 May 2016 14:57:12 +0100 Subject: [PATCH 09/17] Fix line length for PEP8 --- ckanext/example_flask_iroutes/plugin.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ckanext/example_flask_iroutes/plugin.py b/ckanext/example_flask_iroutes/plugin.py index 2edc7fbfe06..ebe68b970e1 100644 --- a/ckanext/example_flask_iroutes/plugin.py +++ b/ckanext/example_flask_iroutes/plugin.py @@ -25,12 +25,14 @@ def override_pylons_about_with_core_template(): def override_flask_hello(): '''A simple replacement for the flash Hello view function.''' html = ''' - - - Hello from Flask - - Hello World, this is served from an extension, overriding the flask url. - ''' + + + Hello from Flask + + + Hello World, this is served from an extension, overriding the flask url. + +''' return render_template_string(html) From ec7754483f0a8c824c493ad91d5722f7370196d2 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Mon, 30 May 2016 16:12:34 +0100 Subject: [PATCH 10/17] Flask extensions should use IBlueprint interface. Flask extensions can register more than just routes for the app. So change the extension point from IRoutes, to IBlueprint, a very simple interface that has one method `get_blueprint`, enabling extensions to return Flask Blueprint objects to be registered with the Flask app at start up. --- ckan/config/middleware.py | 4 ++-- ckan/plugins/interfaces.py | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py index ffa6bb14230..431c59b1506 100644 --- a/ckan/config/middleware.py +++ b/ckan/config/middleware.py @@ -35,7 +35,7 @@ from flask_debugtoolbar import DebugToolbarExtension from ckan.plugins import PluginImplementations -from ckan.plugins.interfaces import IMiddleware, IRoutes +from ckan.plugins.interfaces import IMiddleware, IBlueprint from ckan.lib.i18n import get_locales_from_config import ckan.lib.uploader as uploader from ckan.lib import jinja_extensions @@ -347,7 +347,7 @@ def hello_world_post(): app.register_blueprint(api) # Set up each iRoute extension as a Flask Blueprint - for plugin in PluginImplementations(IRoutes): + for plugin in PluginImplementations(IBlueprint): if hasattr(plugin, 'get_blueprint'): app.register_blueprint(plugin.get_blueprint(), prioritise_rules=True) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index e29d390f132..f013ae9913b 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -1566,3 +1566,11 @@ def get_resource_uploader(self): :type id: string ''' + + +class IBlueprint(Interface): + + '''Register an extension as a Flask Blueprint.''' + + def get_blueprint(self): + '''Return a Flask Blueprint object to be registered by the app.''' From 2796e287a39ce1507aaec55254b618e685bebcb0 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Thu, 2 Jun 2016 12:47:19 +0100 Subject: [PATCH 11/17] Remove Flask-Classy from requirements --- requirements.in | 1 - requirements.txt | 47 ++++++++++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/requirements.in b/requirements.in index b2bc27764e7..4d8d32d9a74 100644 --- a/requirements.in +++ b/requirements.in @@ -34,4 +34,3 @@ tzlocal==1.2.2 wsgi-party==0.1b1 Flask==0.10.1 Flask-Babel==0.10.0 -Flask-Classy==0.6.10 diff --git a/requirements.txt b/requirements.txt index a05886ccf60..c0b763edbd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,38 +1,38 @@ # # This file is autogenerated by pip-compile -# Make changes in requirements.in, then run this to update: +# To update, run: # -# pip-compile requirements.in +# pip-compile --output-file requirements.txt requirements.in # + argparse==1.4.0 # via ofs -Babel==2.3.4 -Beaker==1.7.0 +Babel==2.3.4 # via flask-babel +Beaker==1.7.0 # via pylons bleach==1.4.2 decorator==4.0.6 # via pylons, sqlalchemy-migrate fanstatic==0.12 -Flask==0.10.1 Flask-Babel==0.10.0 -Flask-Classy==0.6.10 -FormEncode==1.3.0 +Flask==0.10.1 # via flask-babel +FormEncode==1.3.0 # via pylons html5lib==0.9999999 # via bleach itsdangerous==0.24 # via flask -Jinja2==2.8 -Mako==1.0.3 +Jinja2==2.8 # via flask, flask-babel +Mako==1.0.3 # via pylons Markdown==2.4 -MarkupSafe==0.23 +MarkupSafe==0.23 # via jinja2, mako, webhelpers nose==1.3.7 # via pylons ofs==0.4.1 ordereddict==1.1 Pairtree==0.7.1-T passlib==1.6.2 paste==1.7.5.1 -PasteDeploy==1.5.2 -PasteScript==2.0.2 +PasteDeploy==1.5.2 # via pastescript, pylons +PasteScript==2.0.2 # via pylons pbr==0.11.1 # via sqlalchemy-migrate psycopg2==2.4.5 -pysolr==3.4.0 -Pygments==2.1 +Pygments==2.1 # via weberror Pylons==0.9.7 +pysolr==3.4.0 python-dateutil==1.5 pytz==2016.4 pyutilib.component.core==4.5.3 @@ -40,21 +40,22 @@ repoze.lru==0.6 # via routes repoze.who-friendlyform==1.0.8 repoze.who==2.0 requests==2.7.0 -Routes==1.13 -simplejson==3.3.1 # via pylons HAND-FIXED FOR NOW #2681 +Routes==1.13 # via pylons +simplejson==3.3.1 # via pylons six==1.10.0 # via bleach, html5lib, pastescript, sqlalchemy-migrate +speaklater==1.3 # via flask-babel sqlalchemy-migrate==0.9.1 -SQLAlchemy==0.9.6 +SQLAlchemy==0.9.6 # via sqlalchemy-migrate sqlparse==0.1.11 -Tempita==0.5.2 +Tempita==0.5.2 # via pylons, sqlalchemy-migrate, weberror tzlocal==1.2.2 unicodecsv==0.14.1 vdm==0.13 -WebError==0.11 -WebHelpers==1.3 -WebOb==1.0.8 -WebTest==1.4.3 -Werkzeug==0.11.3 +WebError==0.11 # via pylons +WebHelpers==1.3 # via pylons +WebOb==1.0.8 # via fanstatic, pylons, repoze.who-friendlyform, weberror, webtest +WebTest==1.4.3 # via pylons +Werkzeug==0.11.3 # via flask, wsgi-party wsgi-party==0.1b1 zope.interface==4.1.1 From 756157f41bf8ae4e53508a228d51a57b36224301 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 3 Jun 2016 14:05:53 +0100 Subject: [PATCH 12/17] Add IBlueprint as an exported interface --- ckan/plugins/interfaces.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ckan/plugins/interfaces.py b/ckan/plugins/interfaces.py index 48586d0aacf..1631ed4e34d 100644 --- a/ckan/plugins/interfaces.py +++ b/ckan/plugins/interfaces.py @@ -25,7 +25,8 @@ 'IFacets', 'IAuthenticator', 'ITranslation', - 'IUploader' + 'IUploader', + 'IBlueprint' ] from inspect import isclass From ff97412708c4d919ea041e2aa617eca6e04cb92b Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 3 Jun 2016 14:06:43 +0100 Subject: [PATCH 13/17] Rename IBlueprint example extension. Rename example_flask_iroutes --> example_flask_blueprint. --- ckan/tests/config/test_middleware.py | 6 +++++- .../__init__.py | 0 .../plugin.py | 7 ++++--- .../templates/about.html | 0 .../templates/about_base.html | 0 .../tests/__init__.py | 0 .../tests/test_routes.py | 8 ++++---- setup.py | 2 +- 8 files changed, 14 insertions(+), 9 deletions(-) rename ckanext/{example_flask_iroutes => example_flask_iblueprint}/__init__.py (100%) rename ckanext/{example_flask_iroutes => example_flask_iblueprint}/plugin.py (93%) rename ckanext/{example_flask_iroutes => example_flask_iblueprint}/templates/about.html (100%) rename ckanext/{example_flask_iroutes => example_flask_iblueprint}/templates/about_base.html (100%) rename ckanext/{example_flask_iroutes => example_flask_iblueprint}/tests/__init__.py (100%) rename ckanext/{example_flask_iroutes => example_flask_iblueprint}/tests/test_routes.py (86%) diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 1dd9b9001cf..d97b22eecbd 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -317,7 +317,7 @@ def test_ask_around_pylons_extension_route_get_after_map(self): def test_ask_around_flask_core_and_pylons_extension_route(self): # TODO: re-enable when we have a way for Flask extensions to add routes - raise nose.SkipTest() + # raise nose.SkipTest() if not p.plugin_loaded('test_routing_plugin'): p.load('test_routing_plugin') @@ -407,6 +407,7 @@ def test_flask_core_and_pylons_core_route_is_served_by_flask(self): class MockRoutingPlugin(p.SingletonPlugin): p.implements(p.IRoutes) + p.implements(p.IBlueprint) controller = 'ckan.tests.config.test_middleware:MockPylonsController' @@ -431,6 +432,9 @@ def after_map(self, _map): return _map + def get_blueprint(self): + pass + class MockPylonsController(p.toolkit.BaseController): diff --git a/ckanext/example_flask_iroutes/__init__.py b/ckanext/example_flask_iblueprint/__init__.py similarity index 100% rename from ckanext/example_flask_iroutes/__init__.py rename to ckanext/example_flask_iblueprint/__init__.py diff --git a/ckanext/example_flask_iroutes/plugin.py b/ckanext/example_flask_iblueprint/plugin.py similarity index 93% rename from ckanext/example_flask_iroutes/plugin.py rename to ckanext/example_flask_iblueprint/plugin.py index ebe68b970e1..78a0af83c82 100644 --- a/ckanext/example_flask_iroutes/plugin.py +++ b/ckanext/example_flask_iblueprint/plugin.py @@ -69,11 +69,12 @@ def helper_here(): return render_template_string(html) -class ExampleFlaskIRoutesPlugin(p.SingletonPlugin): +class ExampleFlaskIBlueprintPlugin(p.SingletonPlugin): ''' - An example IRoutes plugin to demonstrate Flask routing from an extension. + An example IBlueprint plugin to demonstrate Flask routing from an + extension. ''' - p.implements(p.IRoutes, inherit=True) + p.implements(p.IBlueprint, inherit=True) def get_blueprint(self): '''Return a Flask Blueprint object to be registered by the app.''' diff --git a/ckanext/example_flask_iroutes/templates/about.html b/ckanext/example_flask_iblueprint/templates/about.html similarity index 100% rename from ckanext/example_flask_iroutes/templates/about.html rename to ckanext/example_flask_iblueprint/templates/about.html diff --git a/ckanext/example_flask_iroutes/templates/about_base.html b/ckanext/example_flask_iblueprint/templates/about_base.html similarity index 100% rename from ckanext/example_flask_iroutes/templates/about_base.html rename to ckanext/example_flask_iblueprint/templates/about_base.html diff --git a/ckanext/example_flask_iroutes/tests/__init__.py b/ckanext/example_flask_iblueprint/tests/__init__.py similarity index 100% rename from ckanext/example_flask_iroutes/tests/__init__.py rename to ckanext/example_flask_iblueprint/tests/__init__.py diff --git a/ckanext/example_flask_iroutes/tests/test_routes.py b/ckanext/example_flask_iblueprint/tests/test_routes.py similarity index 86% rename from ckanext/example_flask_iroutes/tests/test_routes.py rename to ckanext/example_flask_iblueprint/tests/test_routes.py index 4e72e704ade..cf79f1e59a4 100644 --- a/ckanext/example_flask_iroutes/tests/test_routes.py +++ b/ckanext/example_flask_iblueprint/tests/test_routes.py @@ -4,16 +4,16 @@ import ckan.tests.helpers as helpers -class TestFlaskIRoutes(helpers.FunctionalTestBase): +class TestFlaskIBlueprint(helpers.FunctionalTestBase): def setup(self): self.app = helpers._get_test_app() flask_app = helpers.find_flask_app(self.app) # Install plugin and register its blueprint - if not plugins.plugin_loaded('example_flask_iroutes'): - plugins.load('example_flask_iroutes') - plugin = plugins.get_plugin('example_flask_iroutes') + if not plugins.plugin_loaded('example_flask_iblueprint'): + plugins.load('example_flask_iblueprint') + plugin = plugins.get_plugin('example_flask_iblueprint') flask_app.register_blueprint(plugin.get_blueprint(), prioritise_rules=True) diff --git a/setup.py b/setup.py index dae3f9dedb3..489184820c0 100644 --- a/setup.py +++ b/setup.py @@ -140,7 +140,7 @@ 'example_itranslation = ckanext.example_itranslation.plugin:ExampleITranslationPlugin', 'example_iconfigurer_v1 = ckanext.example_iconfigurer.plugin_v1:ExampleIConfigurerPlugin', 'example_iconfigurer_v2 = ckanext.example_iconfigurer.plugin_v2:ExampleIConfigurerPlugin', - 'example_flask_iroutes = ckanext.example_flask_iroutes.plugin:ExampleFlaskIRoutesPlugin', + 'example_flask_iblueprint = ckanext.example_flask_iblueprint.plugin:ExampleFlaskIBlueprintPlugin', 'example_iuploader = ckanext.example_iuploader.plugin:ExampleIUploader', ], 'ckan.system_plugins': [ From f2be0ababb4351c7b707ac7fdbb251ab7ea558ee Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 3 Jun 2016 14:08:50 +0100 Subject: [PATCH 14/17] Remove `inherit` parameter Nothing to inherit from IBlueprint --- ckanext/example_flask_iblueprint/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/example_flask_iblueprint/plugin.py b/ckanext/example_flask_iblueprint/plugin.py index 78a0af83c82..7b0da3de279 100644 --- a/ckanext/example_flask_iblueprint/plugin.py +++ b/ckanext/example_flask_iblueprint/plugin.py @@ -74,7 +74,7 @@ class ExampleFlaskIBlueprintPlugin(p.SingletonPlugin): An example IBlueprint plugin to demonstrate Flask routing from an extension. ''' - p.implements(p.IBlueprint, inherit=True) + p.implements(p.IBlueprint) def get_blueprint(self): '''Return a Flask Blueprint object to be registered by the app.''' From c75913382597f9d37daafe03b1b22c6a086aded3 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 3 Jun 2016 14:45:24 +0100 Subject: [PATCH 15/17] Fix middleware test for flask extension route --- ckan/tests/config/test_middleware.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index d97b22eecbd..45e8a605776 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -5,6 +5,7 @@ import nose from nose.tools import assert_equals, assert_not_equals, eq_ from routes import url_for +from flask import Blueprint import ckan.plugins as p import ckan.tests.helpers as helpers @@ -314,15 +315,16 @@ def test_ask_around_pylons_extension_route_get_after_map(self): p.unload('test_routing_plugin') - def test_ask_around_flask_core_and_pylons_extension_route(self): + def test_ask_around_flask_extension_and_pylons_extension_route(self): - # TODO: re-enable when we have a way for Flask extensions to add routes - # raise nose.SkipTest() + app = self._get_test_app() + flask_app = helpers.find_flask_app(app) if not p.plugin_loaded('test_routing_plugin'): p.load('test_routing_plugin') - - app = self._get_test_app() + plugin = p.get_plugin('test_routing_plugin') + flask_app.register_blueprint(plugin.get_blueprint(), + prioritise_rules=True) # We want our CKAN app, not the WebTest one app = app.app @@ -433,7 +435,21 @@ def after_map(self, _map): return _map def get_blueprint(self): - pass + # Create Blueprint for plugin + blueprint = Blueprint(self.name, self.__module__) + blueprint.template_folder = 'templates' + # Add plugin url rules to Blueprint object + rules = [ + ('/pylons_and_flask', 'flask_plugin_view', flask_plugin_view), + ] + for rule in rules: + blueprint.add_url_rule(*rule) + + return blueprint + + +def flask_plugin_view(self): + return 'Hello World, this is served from a Flask extension' class MockPylonsController(p.toolkit.BaseController): From 80a83c3a5c3d4cde46b9a74ea28efc3c1892eb4b Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 3 Jun 2016 15:16:55 +0100 Subject: [PATCH 16/17] Neaten up example IBlueprint plugin --- ckan/tests/config/test_middleware.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index 45e8a605776..e8000eebfa1 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -437,13 +437,9 @@ def after_map(self, _map): def get_blueprint(self): # Create Blueprint for plugin blueprint = Blueprint(self.name, self.__module__) - blueprint.template_folder = 'templates' - # Add plugin url rules to Blueprint object - rules = [ - ('/pylons_and_flask', 'flask_plugin_view', flask_plugin_view), - ] - for rule in rules: - blueprint.add_url_rule(*rule) + # Add plugin url rule to Blueprint object + blueprint.add_url_rule('/pylons_and_flask', 'flask_plugin_view', + flask_plugin_view) return blueprint From 2ac658f760ccae9cdcfefbd1e59a64963f766249 Mon Sep 17 00:00:00 2001 From: Brook Elgie Date: Fri, 3 Jun 2016 16:23:53 +0100 Subject: [PATCH 17/17] Refactor middleware module. It was starting to become unmanageably large with a lot of separate concerns in one place. - Separate the flask and pylons app code into separate modules. - Separate the common middleware code into a separate module. --- ckan/config/middleware.py | 770 -------------------- ckan/config/middleware/__init__.py | 132 ++++ ckan/config/middleware/common_middleware.py | 216 ++++++ ckan/config/middleware/flask_app.py | 245 +++++++ ckan/config/middleware/pylons_app.py | 218 ++++++ ckan/tests/config/test_middleware.py | 4 +- ckan/tests/helpers.py | 2 +- 7 files changed, 814 insertions(+), 773 deletions(-) delete mode 100644 ckan/config/middleware.py create mode 100644 ckan/config/middleware/__init__.py create mode 100644 ckan/config/middleware/common_middleware.py create mode 100644 ckan/config/middleware/flask_app.py create mode 100644 ckan/config/middleware/pylons_app.py diff --git a/ckan/config/middleware.py b/ckan/config/middleware.py deleted file mode 100644 index 24f003a2954..00000000000 --- a/ckan/config/middleware.py +++ /dev/null @@ -1,770 +0,0 @@ -# encoding: utf-8 - -"""Pylons middleware initialization""" -import urllib -import urllib2 -import logging -import json -import hashlib -import os -import webob -import itertools - -import sqlalchemy as sa -from beaker.middleware import CacheMiddleware, SessionMiddleware -from paste.cascade import Cascade -from paste.registry import RegistryManager -from paste.urlparser import StaticURLParser -from paste.deploy.converters import asbool -from pylons import config -from pylons.middleware import ErrorHandler, StatusCodeRedirect -from pylons.wsgiapp import PylonsApp -from routes.middleware import RoutesMiddleware -from repoze.who.config import WhoConfig -from repoze.who.middleware import PluggableAuthenticationMiddleware -from fanstatic import Fanstatic - -from wsgi_party import WSGIParty, HighAndDry -from flask import Flask -from flask import abort as flask_abort -from flask import request as flask_request -from flask import _request_ctx_stack -from flask.ctx import _AppCtxGlobals -from flask.sessions import SessionInterface -from werkzeug.exceptions import HTTPException -from werkzeug.test import create_environ, run_wsgi_app -from flask.ext.babel import Babel -from flask_debugtoolbar import DebugToolbarExtension - -from ckan.plugins import PluginImplementations -from ckan.plugins.interfaces import IMiddleware, IBlueprint -from ckan.lib.i18n import get_locales_from_config -import ckan.lib.uploader as uploader -from ckan.lib import jinja_extensions -from ckan.lib import helpers -from ckan.common import c - -from ckan.config.environment import load_environment -import ckan.lib.app_globals as app_globals - -log = logging.getLogger(__name__) - -# This monkey-patches the webob request object because of the way it messes -# with the WSGI environ. - -# Start of webob.requests.BaseRequest monkey patch -original_charset__set = webob.request.BaseRequest._charset__set - - -def custom_charset__set(self, charset): - original_charset__set(self, charset) - if self.environ.get('CONTENT_TYPE', '').startswith(';'): - self.environ['CONTENT_TYPE'] = '' - -webob.request.BaseRequest._charset__set = custom_charset__set - -webob.request.BaseRequest.charset = property( - webob.request.BaseRequest._charset__get, - custom_charset__set, - webob.request.BaseRequest._charset__del, - webob.request.BaseRequest._charset__get.__doc__) - -# End of webob.requests.BaseRequest monkey patch - - -def make_app(conf, full_stack=True, static_files=True, **app_conf): - - # :::TODO::: like the flask app, make the pylons app respond to invites at - # /__invite__/, and handle can_handle_request requests. - - pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) - flask_app = make_flask_stack(conf, **app_conf) - - app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, 'flask_app': flask_app}) - - return app - - -def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): - """Create a Pylons WSGI application and return it - - ``conf`` - The inherited configuration for this application. Normally from - the [DEFAULT] section of the Paste ini file. - - ``full_stack`` - Whether this application provides a full WSGI stack (by default, - meaning it handles its own exceptions and errors). Disable - full_stack when this application is "managed" by another WSGI - middleware. - - ``static_files`` - Whether this application serves its own static files; disable - when another web server is responsible for serving them. - - ``app_conf`` - The application's local configuration. Normally specified in - the [app:] section of the Paste ini file (where - defaults to main). - - """ - # Configure the Pylons environment - load_environment(conf, app_conf) - - # The Pylons WSGI app - app = PylonsApp() - # set pylons globals - app_globals.reset() - - for plugin in PluginImplementations(IMiddleware): - app = plugin.make_middleware(app, config) - - # Routing/Session/Cache Middleware - app = RoutesMiddleware(app, config['routes.map']) - # we want to be able to retrieve the routes middleware to be able to update - # the mapper. We store it in the pylons config to allow this. - config['routes.middleware'] = app - app = SessionMiddleware(app, config) - app = CacheMiddleware(app, config) - - # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) - # app = QueueLogMiddleware(app) - if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): - app = execute_on_completion(app, config, cleanup_pylons_response_string) - - # Fanstatic - if asbool(config.get('debug', False)): - fanstatic_config = { - 'versioning': True, - 'recompute_hashes': True, - 'minified': False, - 'bottom': True, - 'bundle': False, - } - else: - fanstatic_config = { - 'versioning': True, - 'recompute_hashes': False, - 'minified': True, - 'bottom': True, - 'bundle': True, - } - app = Fanstatic(app, **fanstatic_config) - - for plugin in PluginImplementations(IMiddleware): - try: - app = plugin.make_error_log_middleware(app, config) - except AttributeError: - log.critical('Middleware class {0} is missing the method' - 'make_error_log_middleware.'.format(plugin.__class__.__name__)) - - if asbool(full_stack): - # Handle Python exceptions - app = ErrorHandler(app, conf, **config['pylons.errorware']) - - # Display error documents for 400, 403, 404 status codes (and - # 500 when debug is disabled) - if asbool(config['debug']): - app = StatusCodeRedirect(app, [400, 403, 404]) - else: - app = StatusCodeRedirect(app, [400, 403, 404, 500]) - - # Initialize repoze.who - who_parser = WhoConfig(conf['here']) - who_parser.parse(open(app_conf['who.config_file'])) - - app = PluggableAuthenticationMiddleware( - app, - who_parser.identifiers, - who_parser.authenticators, - who_parser.challengers, - who_parser.mdproviders, - who_parser.request_classifier, - who_parser.challenge_decider, - logging.getLogger('repoze.who'), - logging.WARN, # ignored - who_parser.remote_user_key - ) - - # Establish the Registry for this application - app = RegistryManager(app) - - app = I18nMiddleware(app, config) - - if asbool(static_files): - # Serve static files - static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ - else int(config.get('ckan.static_max_age', 3600)) - - static_app = StaticURLParser(config['pylons.paths']['static_files'], - cache_max_age=static_max_age) - static_parsers = [static_app, app] - - storage_directory = uploader.get_storage_path() - if storage_directory: - path = os.path.join(storage_directory, 'storage') - try: - os.makedirs(path) - except OSError, e: - # errno 17 is file already exists - if e.errno != 17: - raise - - storage_app = StaticURLParser(path, cache_max_age=static_max_age) - static_parsers.insert(0, storage_app) - - # Configurable extra static file paths - extra_static_parsers = [] - for public_path in config.get('extra_public_paths', '').split(','): - if public_path.strip(): - extra_static_parsers.append( - StaticURLParser(public_path.strip(), - cache_max_age=static_max_age) - ) - app = Cascade(extra_static_parsers + static_parsers) - - # Page cache - if asbool(config.get('ckan.page_cache_enabled')): - app = PageCacheMiddleware(app, config) - - # Tracking - if asbool(config.get('ckan.tracking_enabled', 'false')): - app = TrackingMiddleware(app, config) - - app = RootPathMiddleware(app, config) - - return app - - -class CKAN_AppCtxGlobals(_AppCtxGlobals): - - '''Custom Flask AppCtxGlobal class (flask.g).''' - - def __getattr__(self, name): - ''' - If flask.g doesn't have attribute `name`, try the app_globals object. - ''' - return getattr(app_globals.app_globals, name) - - -def make_flask_stack(conf, **app_conf): - """ - This passes the flask app through most of the same middleware that Pylons - uses. - """ - - debug = app_conf.get('debug', True) - - root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - app = CKANFlask(__name__) - app.debug = debug - app.template_folder = os.path.join(root, 'templates') - app.app_ctx_globals_class = CKAN_AppCtxGlobals - - # Do all the Flask-specific stuff before adding other middlewares - - # secret key needed for flask-debug-toolbar - app.config['SECRET_KEY'] = '' - app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False - DebugToolbarExtension(app) - - # Use Beaker as the Flask session interface - class BeakerSessionInterface(SessionInterface): - def open_session(self, app, request): - session = request.environ['beaker.session'] - return session - - def save_session(self, app, session, response): - session.save() - - cache_dir = app_conf.get('cache_dir') or app_conf.get('cache.dir') - session_opts = { - 'session.data_dir': '{data_dir}/sessions'.format( - data_dir=cache_dir), - 'session.key': app_conf.get('beaker.session.key'), - 'session.cookie_expires': - app_conf.get('beaker.session.cookie_expires'), - 'session.secret': app_conf.get('beaker.session.secret') - } - app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts) - app.session_interface = BeakerSessionInterface() - - # Add jinja2 extensions and filters - extensions = [ - 'jinja2.ext.do', 'jinja2.ext.with_', - jinja_extensions.SnippetExtension, - jinja_extensions.CkanExtend, - jinja_extensions.CkanInternationalizationExtension, - jinja_extensions.LinkForExtension, - jinja_extensions.ResourceExtension, - jinja_extensions.UrlForStaticExtension, - jinja_extensions.UrlForExtension - ] - for extension in extensions: - app.jinja_env.add_extension(extension) - app.jinja_env.filters['empty_and_escape'] = \ - jinja_extensions.empty_and_escape - app.jinja_env.filters['truncate'] = jinja_extensions.truncate - - # Template context processors - @app.context_processor - def helper_functions(): - helpers.load_plugin_helpers() - return dict(h=helpers.helper_functions) - - @app.context_processor - def c_object(): - return dict(c=c) - - # Babel - app.config['BABEL_TRANSLATION_DIRECTORIES'] = os.path.join( - os.path.dirname(__file__), '..', 'i18n') - app.config['BABEL_DOMAIN'] = 'ckan' - - babel = Babel(app) - - @babel.localeselector - def get_locale(): - ''' - Return the value of the `CKAN_LANG` key of the WSGI environ, - set by the I18nMiddleware based on the URL. - If no value is defined, it defaults to `ckan.locale_default` or `en`. - ''' - from flask import request - return request.environ.get( - 'CKAN_LANG', - config.get('ckan.locale_default', 'en')) - - # A couple of test routes while we migrate to Flask - @app.route('/hello', methods=['GET']) - def hello_world(): - return 'Hello World, this is served by Flask' - - @app.route('/hello', methods=['POST']) - def hello_world_post(): - return 'Hello World, this was posted to Flask' - - # TODO: maybe we can automate this? - from ckan.views.api import api - app.register_blueprint(api) - - # Set up each iRoute extension as a Flask Blueprint - for plugin in PluginImplementations(IBlueprint): - if hasattr(plugin, 'get_blueprint'): - app.register_blueprint(plugin.get_blueprint(), - prioritise_rules=True) - - # Start other middleware - - app = I18nMiddleware(app, config) - - # Initialize repoze.who - who_parser = WhoConfig(conf['here']) - who_parser.parse(open(app_conf['who.config_file'])) - - app = PluggableAuthenticationMiddleware( - app, - who_parser.identifiers, - who_parser.authenticators, - who_parser.challengers, - who_parser.mdproviders, - who_parser.request_classifier, - who_parser.challenge_decider, - logging.getLogger('repoze.who'), - logging.WARN, # ignored - who_parser.remote_user_key - ) - - return app - - -class CKANFlask(Flask): - - '''Extend the Flask class with a special view to join the 'partyline' - established by AskAppDispatcherMiddleware. - - Also provide a 'can_handle_request' method. - ''' - - def __init__(self, import_name, *args, **kwargs): - super(CKANFlask, self).__init__(import_name, *args, **kwargs) - self.add_url_rule('/__invite__/', endpoint='partyline', - view_func=self.join_party) - self.partyline = None - self.partyline_connected = False - self.invitation_context = None - self.app_name = None # A label for the app handling this request - # (this app). - - def join_party(self, request=flask_request): - # Bootstrap, turn the view function into a 404 after registering. - if self.partyline_connected: - # This route does not exist at the HTTP level. - flask_abort(404) - self.invitation_context = _request_ctx_stack.top - self.partyline = request.environ.get(WSGIParty.partyline_key) - self.app_name = request.environ.get('partyline_handling_app') - self.partyline.connect('can_handle_request', self.can_handle_request) - self.partyline_connected = True - return 'ok' - - def can_handle_request(self, environ): - ''' - Decides whether it can handle a request with the Flask app by - matching the request environ against the route mapper - - Returns (True, 'flask_app') if this is the case. - ''' - - # TODO: identify matching urls as core or extension. This will depend - # on how we setup routing in Flask - - urls = self.url_map.bind_to_environ(environ) - try: - endpoint, args = urls.match() - log.debug('Flask route match, endpoint: {0}, args: {1}'.format( - endpoint, args)) - return (True, self.app_name) - except HTTPException: - raise HighAndDry() - - def register_blueprint(self, blueprint, prioritise_rules=False, **options): - ''' - If prioritise_rules is True, add complexity to each url rule in the - blueprint, to ensure they will override similar existing rules. - ''' - - # Register the blueprint with the app. - super(CKANFlask, self).register_blueprint(blueprint, **options) - if prioritise_rules: - # Get the new blueprint rules - bp_rules = [v for k, v in self.url_map._rules_by_endpoint.items() - if k.startswith(blueprint.name)] - bp_rules = list(itertools.chain.from_iterable(bp_rules)) - - # This compare key will ensure the rule will be near the top. - top_compare_key = False, -100, [(-2, 0)] - for r in bp_rules: - r.match_compare_key = lambda: top_compare_key - - -class AskAppDispatcherMiddleware(WSGIParty): - - ''' - Establish a 'partyline' to each provided app. Select which app to call - by asking each if they can handle the requested path at PATH_INFO. - - Used to help transition from Pylons to Flask, and should be removed once - Pylons has been deprecated and all app requests are handled by Flask. - - Each app should handle a call to 'can_handle_request(environ)', responding - with a tuple: - (, , []) - where: - `bool` is True if the app can handle the payload url, - `app` is the wsgi app returning the answer - `origin` is an optional string to determine where in the app the url - will be handled, e.g. 'core' or 'extension'. - - Order of precedence if more than one app can handle a url: - Flask Extension > Pylons Extension > Flask Core > Pylons Core - ''' - - def __init__(self, apps=None, invites=(), ignore_missing_services=False): - # Dict of apps managed by this middleware {: , ...} - self.apps = apps or {} - - # A dict of service name => handler mappings. - self.handlers = {} - - # If True, suppress :class:`NoSuchServiceName` errors. Default: False. - self.ignore_missing_services = ignore_missing_services - - self.send_invitations(apps) - - def send_invitations(self, apps): - '''Call each app at the invite route to establish a partyline. Called - on init.''' - PATH = '/__invite__/' - for app_name, app in apps.items(): - environ = create_environ(path=PATH) - environ[self.partyline_key] = self.operator_class(self) - # A reference to the handling app. Used to id the app when - # responding to a handling request. - environ['partyline_handling_app'] = app_name - run_wsgi_app(app, environ) - - def __call__(self, environ, start_response): - '''Determine which app to call by asking each app if it can handle the - url and method defined on the eviron''' - # :::TODO::: Enforce order of precedence for dispatching to apps here. - - app_name = 'pylons_app' # currently defaulting to pylons app - answers = self.ask_around('can_handle_request', environ) - log.debug('Route support answers for {0} {1}: {2}'.format( - environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), - answers)) - available_handlers = [] - for answer in answers: - if len(answer) == 2: - can_handle, asked_app = answer - origin = 'core' - else: - can_handle, asked_app, origin = answer - if can_handle: - available_handlers.append('{0}_{1}'.format(asked_app, origin)) - - # Enforce order of precedence: - # Flask Extension > Pylons Extension > Flask Core > Pylons Core - if available_handlers: - if 'flask_app_extension' in available_handlers: - app_name = 'flask_app' - elif 'pylons_app_extension' in available_handlers: - app_name = 'pylons_app' - elif 'flask_app_core' in available_handlers: - app_name = 'flask_app' - - log.debug('Serving request via {0} app'.format(app_name)) - environ['ckan.app'] = app_name - return self.apps[app_name](environ, start_response) - - -class RootPathMiddleware(object): - ''' - Prevents the SCRIPT_NAME server variable conflicting with the ckan.root_url - config. The routes package uses the SCRIPT_NAME variable and appends to the - path and ckan addes the root url causing a duplication of the root path. - - This is a middleware to ensure that even redirects use this logic. - ''' - def __init__(self, app, config): - self.app = app - - def __call__(self, environ, start_response): - # Prevents the variable interfering with the root_path logic - if 'SCRIPT_NAME' in environ: - environ['SCRIPT_NAME'] = '' - - return self.app(environ, start_response) - - -class I18nMiddleware(object): - """I18n Middleware selects the language based on the url - eg /fr/home is French""" - def __init__(self, app, config): - self.app = app - self.default_locale = config.get('ckan.locale_default', 'en') - self.local_list = get_locales_from_config() - - def __call__(self, environ, start_response): - # strip the language selector from the requested url - # and set environ variables for the language selected - # CKAN_LANG is the language code eg en, fr - # CKAN_LANG_IS_DEFAULT is set to True or False - # CKAN_CURRENT_URL is set to the current application url - - # We only update once for a request so we can keep - # the language and original url which helps with 404 pages etc - if 'CKAN_LANG' not in environ: - path_parts = environ['PATH_INFO'].split('/') - if len(path_parts) > 1 and path_parts[1] in self.local_list: - environ['CKAN_LANG'] = path_parts[1] - environ['CKAN_LANG_IS_DEFAULT'] = False - # rewrite url - if len(path_parts) > 2: - environ['PATH_INFO'] = '/'.join([''] + path_parts[2:]) - else: - environ['PATH_INFO'] = '/' - else: - environ['CKAN_LANG'] = self.default_locale - environ['CKAN_LANG_IS_DEFAULT'] = True - - # Current application url - path_info = environ['PATH_INFO'] - # sort out weird encodings - path_info = '/'.join(urllib.quote(pce, '') for pce in path_info.split('/')) - - qs = environ.get('QUERY_STRING') - - if qs: - # sort out weird encodings - qs = urllib.quote(qs, '') - environ['CKAN_CURRENT_URL'] = '%s?%s' % (path_info, qs) - else: - environ['CKAN_CURRENT_URL'] = path_info - - return self.app(environ, start_response) - - -class PageCacheMiddleware(object): - ''' A simple page cache that can store and serve pages. It uses - Redis as storage. It caches pages that have a http status code of - 200, use the GET method. Only non-logged in users receive cached - pages. - Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE - variable.''' - - def __init__(self, app, config): - self.app = app - import redis # only import if used - self.redis = redis # we need to reference this within the class - self.redis_exception = redis.exceptions.ConnectionError - self.redis_connection = None - - def __call__(self, environ, start_response): - - def _start_response(status, response_headers, exc_info=None): - # This wrapper allows us to get the status and headers. - environ['CKAN_PAGE_STATUS'] = status - environ['CKAN_PAGE_HEADERS'] = response_headers - return start_response(status, response_headers, exc_info) - - # Only use cache for GET requests - # REMOTE_USER is used by some tests. - if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'): - return self.app(environ, start_response) - - # If there is a ckan cookie (or auth_tkt) we avoid the cache. - # We want to allow other cookies like google analytics ones :( - cookie_string = environ.get('HTTP_COOKIE') - if cookie_string: - for cookie in cookie_string.split(';'): - if cookie.startswith('ckan') or cookie.startswith('auth_tkt'): - return self.app(environ, start_response) - - # Make our cache key - key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING']) - - # Try to connect if we don't have a connection. Doing this here - # allows the redis server to be unavailable at times. - if self.redis_connection is None: - try: - self.redis_connection = self.redis.StrictRedis() - self.redis_connection.flushdb() - except self.redis_exception: - # Connection may have failed at flush so clear it. - self.redis_connection = None - return self.app(environ, start_response) - - # If cached return cached result - try: - result = self.redis_connection.lrange(key, 0, 2) - except self.redis_exception: - # Connection failed so clear it and return the page as normal. - self.redis_connection = None - return self.app(environ, start_response) - - if result: - headers = json.loads(result[1]) - # Convert headers from list to tuples. - headers = [(str(key), str(value)) for key, value in headers] - start_response(str(result[0]), headers) - # Returning a huge string slows down the server. Therefore we - # cut it up into more usable chunks. - page = result[2] - out = [] - total = len(page) - position = 0 - size = 4096 - while position < total: - out.append(page[position:position + size]) - position += size - return out - - # Generate the response from our application. - page = self.app(environ, _start_response) - - # Only cache http status 200 pages - if not environ['CKAN_PAGE_STATUS'].startswith('200'): - return page - - cachable = False - if environ.get('CKAN_PAGE_CACHABLE'): - cachable = True - - # Cache things if cachable. - if cachable: - # Make sure we consume any file handles etc. - page_string = ''.join(list(page)) - # Use a pipe to add page in a transaction. - pipe = self.redis_connection.pipeline() - pipe.rpush(key, environ['CKAN_PAGE_STATUS']) - pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS'])) - pipe.rpush(key, page_string) - pipe.execute() - return page - - -class TrackingMiddleware(object): - - def __init__(self, app, config): - self.app = app - self.engine = sa.create_engine(config.get('sqlalchemy.url')) - - def __call__(self, environ, start_response): - path = environ['PATH_INFO'] - method = environ.get('REQUEST_METHOD') - if path == '/_tracking' and method == 'POST': - # do the tracking - # get the post data - payload = environ['wsgi.input'].read() - parts = payload.split('&') - data = {} - for part in parts: - k, v = part.split('=') - data[k] = urllib2.unquote(v).decode("utf8") - start_response('200 OK', [('Content-Type', 'text/html')]) - # we want a unique anonomized key for each user so that we do - # not count multiple clicks from the same user. - key = ''.join([ - environ['HTTP_USER_AGENT'], - environ['REMOTE_ADDR'], - environ.get('HTTP_ACCEPT_LANGUAGE', ''), - environ.get('HTTP_ACCEPT_ENCODING', ''), - ]) - key = hashlib.md5(key).hexdigest() - # store key/data here - sql = '''INSERT INTO tracking_raw - (user_key, url, tracking_type) - VALUES (%s, %s, %s)''' - self.engine.execute(sql, key, data.get('url'), data.get('type')) - return [] - return self.app(environ, start_response) - - -def generate_close_and_callback(iterable, callback, environ): - """ - return a generator that passes through items from iterable - then calls callback(environ). - """ - try: - for item in iterable: - yield item - except GeneratorExit: - if hasattr(iterable, 'close'): - iterable.close() - raise - finally: - callback(environ) - - -def execute_on_completion(application, config, callback): - """ - Call callback(environ) once complete response is sent - """ - def inner(environ, start_response): - try: - result = application(environ, start_response) - except: - callback(environ) - raise - return generate_close_and_callback(result, callback, environ) - return inner - - -def cleanup_pylons_response_string(environ): - try: - msg = 'response cleared by pylons response cleanup middleware' - environ['pylons.controller']._py_object.response._body = msg - except (KeyError, AttributeError): - pass diff --git a/ckan/config/middleware/__init__.py b/ckan/config/middleware/__init__.py new file mode 100644 index 00000000000..aec57dd9a32 --- /dev/null +++ b/ckan/config/middleware/__init__.py @@ -0,0 +1,132 @@ +# encoding: utf-8 + +"""WSGI app initialization""" + +import webob + +from werkzeug.test import create_environ, run_wsgi_app +from wsgi_party import WSGIParty + +from ckan.config.middleware.flask_app import make_flask_stack +from ckan.config.middleware.pylons_app import make_pylons_stack + +import logging +log = logging.getLogger(__name__) + +# This monkey-patches the webob request object because of the way it messes +# with the WSGI environ. + +# Start of webob.requests.BaseRequest monkey patch +original_charset__set = webob.request.BaseRequest._charset__set + + +def custom_charset__set(self, charset): + original_charset__set(self, charset) + if self.environ.get('CONTENT_TYPE', '').startswith(';'): + self.environ['CONTENT_TYPE'] = '' + +webob.request.BaseRequest._charset__set = custom_charset__set + +webob.request.BaseRequest.charset = property( + webob.request.BaseRequest._charset__get, + custom_charset__set, + webob.request.BaseRequest._charset__del, + webob.request.BaseRequest._charset__get.__doc__) + +# End of webob.requests.BaseRequest monkey patch + + +def make_app(conf, full_stack=True, static_files=True, **app_conf): + ''' + Initialise both the pylons and flask apps, and wrap them in dispatcher + middleware. + ''' + + pylons_app = make_pylons_stack(conf, full_stack, static_files, **app_conf) + flask_app = make_flask_stack(conf, **app_conf) + + app = AskAppDispatcherMiddleware({'pylons_app': pylons_app, + 'flask_app': flask_app}) + + return app + + +class AskAppDispatcherMiddleware(WSGIParty): + + ''' + Establish a 'partyline' to each provided app. Select which app to call + by asking each if they can handle the requested path at PATH_INFO. + + Used to help transition from Pylons to Flask, and should be removed once + Pylons has been deprecated and all app requests are handled by Flask. + + Each app should handle a call to 'can_handle_request(environ)', responding + with a tuple: + (, , []) + where: + `bool` is True if the app can handle the payload url, + `app` is the wsgi app returning the answer + `origin` is an optional string to determine where in the app the url + will be handled, e.g. 'core' or 'extension'. + + Order of precedence if more than one app can handle a url: + Flask Extension > Pylons Extension > Flask Core > Pylons Core + ''' + + def __init__(self, apps=None, invites=(), ignore_missing_services=False): + # Dict of apps managed by this middleware {: , ...} + self.apps = apps or {} + + # A dict of service name => handler mappings. + self.handlers = {} + + # If True, suppress :class:`NoSuchServiceName` errors. Default: False. + self.ignore_missing_services = ignore_missing_services + + self.send_invitations(apps) + + def send_invitations(self, apps): + '''Call each app at the invite route to establish a partyline. Called + on init.''' + PATH = '/__invite__/' + for app_name, app in apps.items(): + environ = create_environ(path=PATH) + environ[self.partyline_key] = self.operator_class(self) + # A reference to the handling app. Used to id the app when + # responding to a handling request. + environ['partyline_handling_app'] = app_name + run_wsgi_app(app, environ) + + def __call__(self, environ, start_response): + '''Determine which app to call by asking each app if it can handle the + url and method defined on the eviron''' + # :::TODO::: Enforce order of precedence for dispatching to apps here. + + app_name = 'pylons_app' # currently defaulting to pylons app + answers = self.ask_around('can_handle_request', environ) + log.debug('Route support answers for {0} {1}: {2}'.format( + environ.get('REQUEST_METHOD'), environ.get('PATH_INFO'), + answers)) + available_handlers = [] + for answer in answers: + if len(answer) == 2: + can_handle, asked_app = answer + origin = 'core' + else: + can_handle, asked_app, origin = answer + if can_handle: + available_handlers.append('{0}_{1}'.format(asked_app, origin)) + + # Enforce order of precedence: + # Flask Extension > Pylons Extension > Flask Core > Pylons Core + if available_handlers: + if 'flask_app_extension' in available_handlers: + app_name = 'flask_app' + elif 'pylons_app_extension' in available_handlers: + app_name = 'pylons_app' + elif 'flask_app_core' in available_handlers: + app_name = 'flask_app' + + log.debug('Serving request via {0} app'.format(app_name)) + environ['ckan.app'] = app_name + return self.apps[app_name](environ, start_response) diff --git a/ckan/config/middleware/common_middleware.py b/ckan/config/middleware/common_middleware.py new file mode 100644 index 00000000000..b141b0f3621 --- /dev/null +++ b/ckan/config/middleware/common_middleware.py @@ -0,0 +1,216 @@ +# encoding: utf-8 + +"""Common middleware used by both Flask and Pylons app stacks.""" + +import urllib2 +import hashlib +import urllib +import json + +import sqlalchemy as sa + +from ckan.lib.i18n import get_locales_from_config + + +class I18nMiddleware(object): + """I18n Middleware selects the language based on the url + eg /fr/home is French""" + def __init__(self, app, config): + self.app = app + self.default_locale = config.get('ckan.locale_default', 'en') + self.local_list = get_locales_from_config() + + def __call__(self, environ, start_response): + # strip the language selector from the requested url + # and set environ variables for the language selected + # CKAN_LANG is the language code eg en, fr + # CKAN_LANG_IS_DEFAULT is set to True or False + # CKAN_CURRENT_URL is set to the current application url + + # We only update once for a request so we can keep + # the language and original url which helps with 404 pages etc + if 'CKAN_LANG' not in environ: + path_parts = environ['PATH_INFO'].split('/') + if len(path_parts) > 1 and path_parts[1] in self.local_list: + environ['CKAN_LANG'] = path_parts[1] + environ['CKAN_LANG_IS_DEFAULT'] = False + # rewrite url + if len(path_parts) > 2: + environ['PATH_INFO'] = '/'.join([''] + path_parts[2:]) + else: + environ['PATH_INFO'] = '/' + else: + environ['CKAN_LANG'] = self.default_locale + environ['CKAN_LANG_IS_DEFAULT'] = True + + # Current application url + path_info = environ['PATH_INFO'] + # sort out weird encodings + path_info = \ + '/'.join(urllib.quote(pce, '') for pce in path_info.split('/')) + + qs = environ.get('QUERY_STRING') + + if qs: + # sort out weird encodings + qs = urllib.quote(qs, '') + environ['CKAN_CURRENT_URL'] = '%s?%s' % (path_info, qs) + else: + environ['CKAN_CURRENT_URL'] = path_info + + return self.app(environ, start_response) + + +class RootPathMiddleware(object): + ''' + Prevents the SCRIPT_NAME server variable conflicting with the ckan.root_url + config. The routes package uses the SCRIPT_NAME variable and appends to the + path and ckan addes the root url causing a duplication of the root path. + + This is a middleware to ensure that even redirects use this logic. + ''' + def __init__(self, app, config): + self.app = app + + def __call__(self, environ, start_response): + # Prevents the variable interfering with the root_path logic + if 'SCRIPT_NAME' in environ: + environ['SCRIPT_NAME'] = '' + + return self.app(environ, start_response) + + +class PageCacheMiddleware(object): + ''' A simple page cache that can store and serve pages. It uses + Redis as storage. It caches pages that have a http status code of + 200, use the GET method. Only non-logged in users receive cached + pages. + Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE + variable.''' + + def __init__(self, app, config): + self.app = app + import redis # only import if used + self.redis = redis # we need to reference this within the class + self.redis_exception = redis.exceptions.ConnectionError + self.redis_connection = None + + def __call__(self, environ, start_response): + + def _start_response(status, response_headers, exc_info=None): + # This wrapper allows us to get the status and headers. + environ['CKAN_PAGE_STATUS'] = status + environ['CKAN_PAGE_HEADERS'] = response_headers + return start_response(status, response_headers, exc_info) + + # Only use cache for GET requests + # REMOTE_USER is used by some tests. + if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'): + return self.app(environ, start_response) + + # If there is a ckan cookie (or auth_tkt) we avoid the cache. + # We want to allow other cookies like google analytics ones :( + cookie_string = environ.get('HTTP_COOKIE') + if cookie_string: + for cookie in cookie_string.split(';'): + if cookie.startswith('ckan') or cookie.startswith('auth_tkt'): + return self.app(environ, start_response) + + # Make our cache key + key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING']) + + # Try to connect if we don't have a connection. Doing this here + # allows the redis server to be unavailable at times. + if self.redis_connection is None: + try: + self.redis_connection = self.redis.StrictRedis() + self.redis_connection.flushdb() + except self.redis_exception: + # Connection may have failed at flush so clear it. + self.redis_connection = None + return self.app(environ, start_response) + + # If cached return cached result + try: + result = self.redis_connection.lrange(key, 0, 2) + except self.redis_exception: + # Connection failed so clear it and return the page as normal. + self.redis_connection = None + return self.app(environ, start_response) + + if result: + headers = json.loads(result[1]) + # Convert headers from list to tuples. + headers = [(str(key), str(value)) for key, value in headers] + start_response(str(result[0]), headers) + # Returning a huge string slows down the server. Therefore we + # cut it up into more usable chunks. + page = result[2] + out = [] + total = len(page) + position = 0 + size = 4096 + while position < total: + out.append(page[position:position + size]) + position += size + return out + + # Generate the response from our application. + page = self.app(environ, _start_response) + + # Only cache http status 200 pages + if not environ['CKAN_PAGE_STATUS'].startswith('200'): + return page + + cachable = False + if environ.get('CKAN_PAGE_CACHABLE'): + cachable = True + + # Cache things if cachable. + if cachable: + # Make sure we consume any file handles etc. + page_string = ''.join(list(page)) + # Use a pipe to add page in a transaction. + pipe = self.redis_connection.pipeline() + pipe.rpush(key, environ['CKAN_PAGE_STATUS']) + pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS'])) + pipe.rpush(key, page_string) + pipe.execute() + return page + + +class TrackingMiddleware(object): + + def __init__(self, app, config): + self.app = app + self.engine = sa.create_engine(config.get('sqlalchemy.url')) + + def __call__(self, environ, start_response): + path = environ['PATH_INFO'] + method = environ.get('REQUEST_METHOD') + if path == '/_tracking' and method == 'POST': + # do the tracking + # get the post data + payload = environ['wsgi.input'].read() + parts = payload.split('&') + data = {} + for part in parts: + k, v = part.split('=') + data[k] = urllib2.unquote(v).decode("utf8") + start_response('200 OK', [('Content-Type', 'text/html')]) + # we want a unique anonomized key for each user so that we do + # not count multiple clicks from the same user. + key = ''.join([ + environ['HTTP_USER_AGENT'], + environ['REMOTE_ADDR'], + environ.get('HTTP_ACCEPT_LANGUAGE', ''), + environ.get('HTTP_ACCEPT_ENCODING', ''), + ]) + key = hashlib.md5(key).hexdigest() + # store key/data here + sql = '''INSERT INTO tracking_raw + (user_key, url, tracking_type) + VALUES (%s, %s, %s)''' + self.engine.execute(sql, key, data.get('url'), data.get('type')) + return [] + return self.app(environ, start_response) diff --git a/ckan/config/middleware/flask_app.py b/ckan/config/middleware/flask_app.py new file mode 100644 index 00000000000..041ba964aa6 --- /dev/null +++ b/ckan/config/middleware/flask_app.py @@ -0,0 +1,245 @@ +# encoding: utf-8 + +import os +import itertools + +from flask import Flask +from flask import abort as flask_abort +from flask import request as flask_request +from flask import _request_ctx_stack +from flask.ctx import _AppCtxGlobals +from flask.sessions import SessionInterface +from werkzeug.exceptions import HTTPException + +from wsgi_party import WSGIParty, HighAndDry +from flask.ext.babel import Babel +from flask_debugtoolbar import DebugToolbarExtension +from pylons import config + +from beaker.middleware import SessionMiddleware +from repoze.who.config import WhoConfig +from repoze.who.middleware import PluggableAuthenticationMiddleware + +import ckan.lib.app_globals as app_globals +from ckan.lib import jinja_extensions +from ckan.lib import helpers +from ckan.common import c +from ckan.plugins import PluginImplementations +from ckan.plugins.interfaces import IBlueprint + +from ckan.config.middleware import common_middleware + +import logging +log = logging.getLogger(__name__) + + +def make_flask_stack(conf, **app_conf): + """ + This passes the flask app through most of the same middleware that Pylons + uses. + """ + + debug = app_conf.get('debug', True) + + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + app = CKANFlask(__name__) + app.debug = debug + app.template_folder = os.path.join(root, 'templates') + app.app_ctx_globals_class = CKAN_AppCtxGlobals + + # Do all the Flask-specific stuff before adding other middlewares + + # secret key needed for flask-debug-toolbar + app.config['SECRET_KEY'] = '' + app.config['DEBUG_TB_INTERCEPT_REDIRECTS'] = False + DebugToolbarExtension(app) + + # Use Beaker as the Flask session interface + class BeakerSessionInterface(SessionInterface): + def open_session(self, app, request): + session = request.environ['beaker.session'] + return session + + def save_session(self, app, session, response): + session.save() + + cache_dir = app_conf.get('cache_dir') or app_conf.get('cache.dir') + session_opts = { + 'session.data_dir': '{data_dir}/sessions'.format( + data_dir=cache_dir), + 'session.key': app_conf.get('beaker.session.key'), + 'session.cookie_expires': + app_conf.get('beaker.session.cookie_expires'), + 'session.secret': app_conf.get('beaker.session.secret') + } + app.wsgi_app = SessionMiddleware(app.wsgi_app, session_opts) + app.session_interface = BeakerSessionInterface() + + # Add jinja2 extensions and filters + extensions = [ + 'jinja2.ext.do', 'jinja2.ext.with_', + jinja_extensions.SnippetExtension, + jinja_extensions.CkanExtend, + jinja_extensions.CkanInternationalizationExtension, + jinja_extensions.LinkForExtension, + jinja_extensions.ResourceExtension, + jinja_extensions.UrlForStaticExtension, + jinja_extensions.UrlForExtension + ] + for extension in extensions: + app.jinja_env.add_extension(extension) + app.jinja_env.filters['empty_and_escape'] = \ + jinja_extensions.empty_and_escape + app.jinja_env.filters['truncate'] = jinja_extensions.truncate + + # Template context processors + @app.context_processor + def helper_functions(): + helpers.load_plugin_helpers() + return dict(h=helpers.helper_functions) + + @app.context_processor + def c_object(): + return dict(c=c) + + # Babel + app.config['BABEL_TRANSLATION_DIRECTORIES'] = os.path.join( + os.path.dirname(__file__), '..', 'i18n') + app.config['BABEL_DOMAIN'] = 'ckan' + + babel = Babel(app) + + @babel.localeselector + def get_locale(): + ''' + Return the value of the `CKAN_LANG` key of the WSGI environ, + set by the I18nMiddleware based on the URL. + If no value is defined, it defaults to `ckan.locale_default` or `en`. + ''' + from flask import request + return request.environ.get( + 'CKAN_LANG', + config.get('ckan.locale_default', 'en')) + + # A couple of test routes while we migrate to Flask + @app.route('/hello', methods=['GET']) + def hello_world(): + return 'Hello World, this is served by Flask' + + @app.route('/hello', methods=['POST']) + def hello_world_post(): + return 'Hello World, this was posted to Flask' + + # TODO: maybe we can automate this? + from ckan.views.api import api + app.register_blueprint(api) + + # Set up each iRoute extension as a Flask Blueprint + for plugin in PluginImplementations(IBlueprint): + if hasattr(plugin, 'get_blueprint'): + app.register_blueprint(plugin.get_blueprint(), + prioritise_rules=True) + + # Start other middleware + + app = common_middleware.I18nMiddleware(app, config) + + # Initialize repoze.who + who_parser = WhoConfig(conf['here']) + who_parser.parse(open(app_conf['who.config_file'])) + + app = PluggableAuthenticationMiddleware( + app, + who_parser.identifiers, + who_parser.authenticators, + who_parser.challengers, + who_parser.mdproviders, + who_parser.request_classifier, + who_parser.challenge_decider, + logging.getLogger('repoze.who'), + logging.WARN, # ignored + who_parser.remote_user_key + ) + + return app + + +class CKAN_AppCtxGlobals(_AppCtxGlobals): + + '''Custom Flask AppCtxGlobal class (flask.g).''' + + def __getattr__(self, name): + ''' + If flask.g doesn't have attribute `name`, try the app_globals object. + ''' + return getattr(app_globals.app_globals, name) + + +class CKANFlask(Flask): + + '''Extend the Flask class with a special view to join the 'partyline' + established by AskAppDispatcherMiddleware. + + Also provide a 'can_handle_request' method. + ''' + + def __init__(self, import_name, *args, **kwargs): + super(CKANFlask, self).__init__(import_name, *args, **kwargs) + self.add_url_rule('/__invite__/', endpoint='partyline', + view_func=self.join_party) + self.partyline = None + self.partyline_connected = False + self.invitation_context = None + # A label for the app handling this request (this app). + self.app_name = None + + def join_party(self, request=flask_request): + # Bootstrap, turn the view function into a 404 after registering. + if self.partyline_connected: + # This route does not exist at the HTTP level. + flask_abort(404) + self.invitation_context = _request_ctx_stack.top + self.partyline = request.environ.get(WSGIParty.partyline_key) + self.app_name = request.environ.get('partyline_handling_app') + self.partyline.connect('can_handle_request', self.can_handle_request) + self.partyline_connected = True + return 'ok' + + def can_handle_request(self, environ): + ''' + Decides whether it can handle a request with the Flask app by + matching the request environ against the route mapper + + Returns (True, 'flask_app') if this is the case. + ''' + + # TODO: identify matching urls as core or extension. This will depend + # on how we setup routing in Flask + + urls = self.url_map.bind_to_environ(environ) + try: + endpoint, args = urls.match() + log.debug('Flask route match, endpoint: {0}, args: {1}'.format( + endpoint, args)) + return (True, self.app_name) + except HTTPException: + raise HighAndDry() + + def register_blueprint(self, blueprint, prioritise_rules=False, **options): + ''' + If prioritise_rules is True, add complexity to each url rule in the + blueprint, to ensure they will override similar existing rules. + ''' + + # Register the blueprint with the app. + super(CKANFlask, self).register_blueprint(blueprint, **options) + if prioritise_rules: + # Get the new blueprint rules + bp_rules = [v for k, v in self.url_map._rules_by_endpoint.items() + if k.startswith(blueprint.name)] + bp_rules = list(itertools.chain.from_iterable(bp_rules)) + + # This compare key will ensure the rule will be near the top. + top_compare_key = False, -100, [(-2, 0)] + for r in bp_rules: + r.match_compare_key = lambda: top_compare_key diff --git a/ckan/config/middleware/pylons_app.py b/ckan/config/middleware/pylons_app.py new file mode 100644 index 00000000000..4d2af590c0c --- /dev/null +++ b/ckan/config/middleware/pylons_app.py @@ -0,0 +1,218 @@ +# encoding: utf-8 + +import os + +from pylons import config +from pylons.wsgiapp import PylonsApp + +from beaker.middleware import CacheMiddleware, SessionMiddleware +from paste.cascade import Cascade +from paste.registry import RegistryManager +from paste.urlparser import StaticURLParser +from paste.deploy.converters import asbool +from pylons.middleware import ErrorHandler, StatusCodeRedirect +from routes.middleware import RoutesMiddleware +from repoze.who.config import WhoConfig +from repoze.who.middleware import PluggableAuthenticationMiddleware +from fanstatic import Fanstatic + +from ckan.plugins import PluginImplementations +from ckan.plugins.interfaces import IMiddleware +import ckan.lib.uploader as uploader +from ckan.config.environment import load_environment +import ckan.lib.app_globals as app_globals +from ckan.config.middleware import common_middleware + +import logging +log = logging.getLogger(__name__) + + +def make_pylons_stack(conf, full_stack=True, static_files=True, **app_conf): + """Create a Pylons WSGI application and return it + + ``conf`` + The inherited configuration for this application. Normally from + the [DEFAULT] section of the Paste ini file. + + ``full_stack`` + Whether this application provides a full WSGI stack (by default, + meaning it handles its own exceptions and errors). Disable + full_stack when this application is "managed" by another WSGI + middleware. + + ``static_files`` + Whether this application serves its own static files; disable + when another web server is responsible for serving them. + + ``app_conf`` + The application's local configuration. Normally specified in + the [app:] section of the Paste ini file (where + defaults to main). + + """ + # Configure the Pylons environment + load_environment(conf, app_conf) + + # The Pylons WSGI app + app = PylonsApp() + # set pylons globals + app_globals.reset() + + for plugin in PluginImplementations(IMiddleware): + app = plugin.make_middleware(app, config) + + # Routing/Session/Cache Middleware + app = RoutesMiddleware(app, config['routes.map']) + # we want to be able to retrieve the routes middleware to be able to update + # the mapper. We store it in the pylons config to allow this. + config['routes.middleware'] = app + app = SessionMiddleware(app, config) + app = CacheMiddleware(app, config) + + # CUSTOM MIDDLEWARE HERE (filtered by error handling middlewares) + # app = QueueLogMiddleware(app) + if asbool(config.get('ckan.use_pylons_response_cleanup_middleware', True)): + app = execute_on_completion(app, config, + cleanup_pylons_response_string) + + # Fanstatic + if asbool(config.get('debug', False)): + fanstatic_config = { + 'versioning': True, + 'recompute_hashes': True, + 'minified': False, + 'bottom': True, + 'bundle': False, + } + else: + fanstatic_config = { + 'versioning': True, + 'recompute_hashes': False, + 'minified': True, + 'bottom': True, + 'bundle': True, + } + app = Fanstatic(app, **fanstatic_config) + + for plugin in PluginImplementations(IMiddleware): + try: + app = plugin.make_error_log_middleware(app, config) + except AttributeError: + log.critical('Middleware class {0} is missing the method' + 'make_error_log_middleware.' + .format(plugin.__class__.__name__)) + + if asbool(full_stack): + # Handle Python exceptions + app = ErrorHandler(app, conf, **config['pylons.errorware']) + + # Display error documents for 400, 403, 404 status codes (and + # 500 when debug is disabled) + if asbool(config['debug']): + app = StatusCodeRedirect(app, [400, 403, 404]) + else: + app = StatusCodeRedirect(app, [400, 403, 404, 500]) + + # Initialize repoze.who + who_parser = WhoConfig(conf['here']) + who_parser.parse(open(app_conf['who.config_file'])) + + app = PluggableAuthenticationMiddleware( + app, + who_parser.identifiers, + who_parser.authenticators, + who_parser.challengers, + who_parser.mdproviders, + who_parser.request_classifier, + who_parser.challenge_decider, + logging.getLogger('repoze.who'), + logging.WARN, # ignored + who_parser.remote_user_key + ) + + # Establish the Registry for this application + app = RegistryManager(app) + + app = common_middleware.I18nMiddleware(app, config) + + if asbool(static_files): + # Serve static files + static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \ + else int(config.get('ckan.static_max_age', 3600)) + + static_app = StaticURLParser(config['pylons.paths']['static_files'], + cache_max_age=static_max_age) + static_parsers = [static_app, app] + + storage_directory = uploader.get_storage_path() + if storage_directory: + path = os.path.join(storage_directory, 'storage') + try: + os.makedirs(path) + except OSError, e: + # errno 17 is file already exists + if e.errno != 17: + raise + + storage_app = StaticURLParser(path, cache_max_age=static_max_age) + static_parsers.insert(0, storage_app) + + # Configurable extra static file paths + extra_static_parsers = [] + for public_path in config.get('extra_public_paths', '').split(','): + if public_path.strip(): + extra_static_parsers.append( + StaticURLParser(public_path.strip(), + cache_max_age=static_max_age) + ) + app = Cascade(extra_static_parsers + static_parsers) + + # Page cache + if asbool(config.get('ckan.page_cache_enabled')): + app = common_middleware.PageCacheMiddleware(app, config) + + # Tracking + if asbool(config.get('ckan.tracking_enabled', 'false')): + app = common_middleware.TrackingMiddleware(app, config) + + app = common_middleware.RootPathMiddleware(app, config) + + return app + + +def generate_close_and_callback(iterable, callback, environ): + """ + return a generator that passes through items from iterable + then calls callback(environ). + """ + try: + for item in iterable: + yield item + except GeneratorExit: + if hasattr(iterable, 'close'): + iterable.close() + raise + finally: + callback(environ) + + +def execute_on_completion(application, config, callback): + """ + Call callback(environ) once complete response is sent + """ + def inner(environ, start_response): + try: + result = application(environ, start_response) + except: + callback(environ) + raise + return generate_close_and_callback(result, callback, environ) + return inner + + +def cleanup_pylons_response_string(environ): + try: + msg = 'response cleared by pylons response cleanup middleware' + environ['pylons.controller']._py_object.response._body = msg + except (KeyError, AttributeError): + pass diff --git a/ckan/tests/config/test_middleware.py b/ckan/tests/config/test_middleware.py index e8000eebfa1..0ef90266351 100644 --- a/ckan/tests/config/test_middleware.py +++ b/ckan/tests/config/test_middleware.py @@ -2,7 +2,6 @@ import mock import wsgiref -import nose from nose.tools import assert_equals, assert_not_equals, eq_ from routes import url_for from flask import Blueprint @@ -10,7 +9,8 @@ import ckan.plugins as p import ckan.tests.helpers as helpers -from ckan.config.middleware import AskAppDispatcherMiddleware, CKANFlask +from ckan.config.middleware import AskAppDispatcherMiddleware +from ckan.config.middleware.flask_app import CKANFlask from ckan.controllers.partyline import PartylineController diff --git a/ckan/tests/helpers.py b/ckan/tests/helpers.py index 1535e206050..998d0fef04a 100644 --- a/ckan/tests/helpers.py +++ b/ckan/tests/helpers.py @@ -441,7 +441,7 @@ def find_flask_app(test_app): Relies on each layer of the stack having a reference to the app they wrap in either a .app attribute or .apps list. ''' - if isinstance(test_app, ckan.config.middleware.CKANFlask): + if isinstance(test_app, ckan.config.middleware.flask_app.CKANFlask): return test_app try: