Merge branch 'enhancement-2220-cache-friendly'

ckan/config/middleware.py

@@ -1,6 +1,7 @@
 """Pylons middleware initialization"""
 import urllib
 import logging 
+import json
 
 from beaker.middleware import CacheMiddleware, SessionMiddleware
 from paste.cascade import Cascade
@@ -101,10 +102,11 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 who_parser.remote_user_key,
            )
 
-    app = I18nMiddleware(app, config)
     # Establish the Registry for this application
     app = RegistryManager(app)
 
+    app = I18nMiddleware(app, config)
+
     if asbool(static_files):
         # Serve static files
         static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \
@@ -124,6 +126,10 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 )
         app = Cascade(extra_static_parsers+static_parsers)
 
+    # Page cache
+    if asbool(config.get('ckan.page_cache_enabled')):
+        app = PageCacheMiddleware(app, config)
+
     return app
 
 class I18nMiddleware(object):
@@ -134,17 +140,6 @@ def __init__(self, app, config):
         self.default_locale = config.get('ckan.locale_default', 'en')
         self.local_list = get_locales()
 
-    def get_cookie_lang(self, environ):
-        # get the lang from cookie if present
-        cookie = environ.get('HTTP_COOKIE')
-        if cookie:
-            cookies = [c.strip() for c in cookie.split(';')]
-            lang = [c.split('=')[1] for c in cookies \
-                    if c.startswith('ckan_lang')]
-            if lang and lang[0] in self.local_list:
-                return lang[0]
-        return None
-
     def __call__(self, environ, start_response):
         # strip the language selector from the requested url
         # and set environ variables for the language selected
@@ -165,16 +160,8 @@ def __call__(self, environ, start_response):
                 else:
                     environ['PATH_INFO'] = '/'
             else:
-                # use cookie lang or default language from config
-                cookie_lang = self.get_cookie_lang(environ)
-                if cookie_lang:
-                    environ['CKAN_LANG'] = cookie_lang
-                    default = (cookie_lang == self.default_locale)
-                    environ['CKAN_LANG_IS_DEFAULT'] = default
-                else:
-                    environ['CKAN_LANG'] = self.default_locale
-                    environ['CKAN_LANG_IS_DEFAULT'] = True
-
+                environ['CKAN_LANG'] = self.default_locale
+                environ['CKAN_LANG_IS_DEFAULT'] = True
 
             # Current application url
             path_info = environ['PATH_INFO']
@@ -191,3 +178,94 @@ def __call__(self, environ, start_response):
                 environ['CKAN_CURRENT_URL'] = path_info
 
         return self.app(environ, start_response)
+
+
+class PageCacheMiddleware(object):
+    ''' A simple page cache that can store and serve pages. It uses
+    Redis as storage. It caches pages that have a http status code of
+    200, use the GET method. Only non-logged in users receive cached
+    pages.
+    Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE
+    variable.'''
+
+    def __init__(self, app, config):
+        self.app = app
+        import redis    # only import if used
+        self.redis = redis  # we need to reference this within the class
+        self.redis_exception = redis.exceptions.ConnectionError
+        self.redis_connection = None
+
+    def __call__(self, environ, start_response):
+
+        def _start_response(status, response_headers, exc_info=None):
+            # This wrapper allows us to get the status and headers.
+            environ['CKAN_PAGE_STATUS'] = status
+            environ['CKAN_PAGE_HEADERS'] = response_headers
+            return start_response(status, response_headers, exc_info)
+
+        # Only use cache for GET requests
+        # If there is a cookie we avoid the cache.
+        # REMOTE_USER is used by some tests.
+        if environ['REQUEST_METHOD'] != 'GET' or environ.get('HTTP_COOKIE') or \
+           environ.get('REMOTE_USER'):
+            return self.app(environ, start_response)
+
+        # Make our cache key
+        key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING'])
+
+        # Try to connect if we don't have a connection. Doing this here
+        # allows the redis server to be unavailable at times.
+        if self.redis_connection is None:
+            try:
+                self.redis_connection = self.redis.StrictRedis()
+                self.redis_connection.flushdb()
+            except self.redis_exception:
+                return self.app(environ, start_response)
+
+        # If cached return cached result
+        try:
+            result = self.redis_connection.lrange(key, 0, 2)
+        except self.redis_exception:
+            # Connection failed so clear it and return the page as normal.
+            self.redis_connection = None
+            return self.app(environ, start_response)
+
+        if result:
+            headers = json.loads(result[1])
+            # Convert headers from list to tuples.
+            headers = [(str(key), str(value)) for key, value in headers]
+            start_response(str(result[0]), headers)
+            # Returning a huge string slows down the server. Therefore we
+            # cut it up into more usable chunks.
+            page = result[2]
+            out = []
+            total = len(page)
+            position = 0
+            size = 4096
+            while position < total:
+                out.append(page[position:position + size])
+                position += size
+            return out
+
+        # Generate the response from our application.
+        page = self.app(environ, _start_response)
+
+        # Only cache http status 200 pages
+        if not environ['CKAN_PAGE_STATUS'].startswith('200'):
+            return page
+
+        cachable = False
+        if environ.get('CKAN_PAGE_CACHABLE'):
+            cachable = True
+
+        # Cache things if cachable.
+        if cachable:
+            # Make sure we consume any file handles etc.
+            page_string = ''.join(list(page))
+            # Use a pipe to add page in a transaction.
+            pipe = self.redis_connection.pipeline()
+            pipe.rpush(key, environ['CKAN_PAGE_STATUS'])
+            pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS']))
+            pipe.rpush(key, page_string)
+            pipe.execute()
+        return page

ckan/config/routing.py

@@ -25,6 +25,7 @@ def make_map():
     DELETE = dict(method=['DELETE'])
     GET_POST = dict(method=['GET', 'POST'])
     PUT_POST = dict(method=['PUT','POST'])
+    GET_POST_DELETE = dict(method=['GET', 'POST', 'DELETE'])
     OPTIONS = dict(method=['OPTIONS'])
 
     from ckan.lib.plugins import register_package_plugins
@@ -104,8 +105,8 @@ def make_map():
         m.connect('/rest/{register}/{id}/:subregister/{id2}', action='delete',
             conditions=DELETE)
 
-    # /api/2/util
-    with SubMapper(map, controller='api', path_prefix='/api{ver:/2}', ver='/2') as m:
+    # /api/util ver 1, 2 or none
+    with SubMapper(map, controller='api', path_prefix='/api{ver:/1|/2|}', ver='/1') as m:
         m.connect('/util/user/autocomplete', action='user_autocomplete')
         m.connect('/util/is_slug_valid', action='is_slug_valid',
                   conditions=GET)
@@ -120,32 +121,27 @@ def make_map():
         m.connect('/util/authorizationgroup/autocomplete',
                   action='authorizationgroup_autocomplete')
         m.connect('/util/group/autocomplete', action='group_autocomplete')
-
-    # /api/util
-    with SubMapper(map, controller='api', path_prefix='/api') as m:
         m.connect('/util/markdown', action='markdown')
         m.connect('/util/dataset/munge_name', action='munge_package_name')
         m.connect('/util/dataset/munge_title_to_name',
                   action='munge_title_to_package_name')
         m.connect('/util/tag/munge', action='munge_tag')
         m.connect('/util/status', action='status')
 
-    ## Webstore
-    if config.get('ckan.datastore.enabled', False):
-        map.connect('datastore_read', '/api/data/{id}{url:(/.*)?}',
-            controller='datastore', action='read', url='',
-            conditions={'method': ['GET']}
-            )
-        map.connect('datastore_write', '/api/data/{id}{url:(/.*)?}',
-            controller='datastore', action='write', url='',
-            conditions={'method': ['PUT','POST', 'DELETE']}
-            )
-
-
     ###########
     ## /END API
     ###########
 
+
+    ## Webstore
+    if config.get('ckan.datastore.enabled', False):
+        with SubMapper(map, controller='datastore') as m:
+            m.connect('datastore_read', '/api/data/{id}{url:(/.*)?}',
+                      action='read', url='', conditions=GET)
+            m.connect('datastore_write', '/api/data/{id}{url:(/.*)?}',
+                      action='write', url='', conditions=GET_POST_DELETE)
+
+
     map.redirect('/packages', '/dataset')
     map.redirect('/packages/{url:.*}', '/dataset/{url}')
     map.redirect('/package', '/dataset')
@@ -245,8 +241,10 @@ def make_map():
         m.connect('/user/login', action='login')
         m.connect('/user/logged_in', action='logged_in')
         m.connect('/user/logged_out', action='logged_out')
+        m.connect('/user/logged_out_redirect', action='logged_out_page')
         m.connect('/user/reset', action='request_reset')
         m.connect('/user/me', action='me')
+        m.connect('/user/set_lang/{lang}', action='set_lang')
         m.connect('/user/{id:.*}', action='read')
         m.connect('/user', action='index')
 

ckan/controllers/admin.py

@@ -241,7 +241,7 @@ def trash(self):
                 model.Revision).filter_by(state=model.State.DELETED)
         c.deleted_packages = model.Session.query(
                 model.Package).filter_by(state=model.State.DELETED)
-        if not request.params:
+        if not request.params or (len(request.params) == 1 and '__no_cache__' in request.params):
             return render('admin/trash.html')
         else:
             # NB: we repeat retrieval of of revisions