fix merge conflicts and search error

README.rst

@@ -12,12 +12,12 @@ http://ckan.org/.
 Building Documentation
 ======================
 
-1. Install python-sphinx
+1. Install python-sphinx (>= 1.1)
 
 2. Initialize the theme submodule::
 
-   git submodule init
-   git submodule update
+    git submodule init
+    git submodule update
 
 3. Run the command to build the docs::
 

ckan/ckan_nose_plugin.py

@@ -1,10 +1,14 @@
 from nose.plugins import Plugin
 from inspect import isclass
+import hashlib
 import os
 import sys
+import re
 import pkg_resources
 from paste.deploy import loadapp
 from pylons import config
+import unittest
+import time
 
 class CkanNose(Plugin):
     settings = None
@@ -53,15 +57,61 @@ def options(self, parser, env):
             action='store_true',
             dest='docstrings',
             help='set this to display test docstrings instead of module names')
+        parser.add_option(
+            '--segments',
+            dest='segments',
+            help='A string containing a hex digits that represent which of'
+                 'the 16 test segments to run. i.e 15af will run segments 1,5,a,f')
+
+    def wantClass(self, cls):
+        name = cls.__name__
+
+        wanted = (not cls.__name__.startswith('_')
+                  and (issubclass(cls, unittest.TestCase)
+                       or re.search('(?:^|[\b_\./-])[Tt]est', name)
+                      ))
+
+        if self.segments and str(hashlib.md5(name).hexdigest())[0] not in self.segments:
+            return False
+
+        return wanted
+
+    def finalize(self, report):
+        if self.segments:
+            print 'Segments: %s' % self.segments
 
     def configure(self, settings, config):
         CkanNose.settings = settings
         if settings.is_ckan:
             self.enabled = True
             self.is_first_test = True
+        self.segments = settings.segments
 
     def describeTest(self, test):
         if not CkanNose.settings.docstrings:
             # display module name instead of docstring
             return False
 
+    def startTest(self, test):
+        """
+        startTest: start timing.
+        """
+##        self._started = time.time()
+
+    def stopTest(self, test):
+        """
+        stopTest: stop timing, canonicalize the test name, and save
+        the run time.
+        """
+##        runtime = time.time() - self._started
+##
+##        # CTB: HACK!
+##        f = open('times.txt', 'a')
+##
+##        testname = str(test)
+##        #if ' ' in testname:
+##        #    testname = testname.split(' ')[1]
+##
+##        f.write('%s,%s\n' % (testname, str(runtime)))
+##
+##        f.close()

ckan/config/environment.py

@@ -7,9 +7,12 @@
 from paste.deploy.converters import asbool
 
 # Suppress benign warning 'Unbuilt egg for setuptools'
-warnings.simplefilter('ignore', UserWarning) 
+warnings.simplefilter('ignore', UserWarning)
+
+
 import pylons
-from sqlalchemy import engine_from_config
+import sqlalchemy
+
 from pylons import config
 from pylons.i18n.translation import ugettext
 from genshi.template import TemplateLoader
@@ -27,9 +30,9 @@ def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
     object
     """
-    
+
     ######  Pylons monkey-patch
-    # this must be run at a time when the env is semi-setup, thus inlined here. 
+    # this must be run at a time when the env is semi-setup, thus inlined here.
     # Required by the deliverance plugin and iATI
     from pylons.wsgiapp import PylonsApp
     import pkg_resources
@@ -45,7 +48,7 @@ def find_controller(self, controller):
             return mycontroller
         return find_controller_generic(self, controller)
     PylonsApp.find_controller = find_controller
-    ###### END evil monkey-patch 
+    ###### END evil monkey-patch
 
     os.environ['CKAN_CONFIG'] = global_conf['__file__']
 
@@ -57,21 +60,25 @@ def find_controller(self, controller):
                  templates=[os.path.join(root, 'templates')])
 
     # Initialize config with the basic options
-    
+
     config.init_app(global_conf, app_conf, package='ckan', paths=paths)
-    
+
     # load all CKAN plugins
     plugins.load_all(config)
 
     from ckan.plugins import PluginImplementations
     from ckan.plugins.interfaces import IConfigurer
-    
+
     for plugin in PluginImplementations(IConfigurer):
         # must do update in place as this does not work:
         # config = plugin.update_config(config)
         plugin.update_config(config)
-    
+
     # This is set up before globals are initialized
+    site_id = os.environ.get('CKAN_SITE_ID')
+    if site_id:
+        config['ckan.site_id'] = site_id
+
     site_url = config.get('ckan.site_url', '')
     ckan_host = config['ckan.host'] = urlparse(site_url).netloc
     if config.get('ckan.site_id') is None:
@@ -91,7 +98,7 @@ def find_controller(self, controller):
     config['routes.map'] = make_map()
     config['pylons.app_globals'] = app_globals.Globals()
     config['pylons.h'] = ckan.lib.helpers
-        
+
     ## redo template setup to use genshi.search_path (so remove std template setup)
     template_paths = [paths['templates'][0]]
     extra_template_paths = config.get('extra_template_paths', '')
@@ -110,24 +117,31 @@ def template_loaded(template):
     logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
 
     # Create the Genshi TemplateLoader
-    # config['pylons.app_globals'].genshi_loader = TemplateLoader(
-    #    paths['templates'], auto_reload=True)
-    # tmpl_options["genshi.loader_callback"] = template_loaded
     config['pylons.app_globals'].genshi_loader = TemplateLoader(
         template_paths, auto_reload=True, callback=template_loaded)
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
-    # any Pylons config options)    
+    # any Pylons config options)
 
     # Setup the SQLAlchemy database engine
-    engine = engine_from_config(config, 'sqlalchemy.')
+    # Suppress a couple of sqlalchemy warnings
+    warnings.filterwarnings('ignore', '^Unicode type received non-unicode bind param value', sqlalchemy.exc.SAWarning)
+    warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning)
+    warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning)
+
+    ckan_db = os.environ.get('CKAN_DB') 
+
+    if ckan_db:
+        engine = sqlalchemy.create_engine(ckan_db)
+    else:
+        engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
 
     if not model.meta.engine:
         model.init_model(engine)
-    
+
     from ckan.plugins import PluginImplementations
     from ckan.plugins.interfaces import IConfigurable
-    
+
     for plugin in PluginImplementations(IConfigurable):
         plugin.configure(config)
-    
+

ckan/config/middleware.py

@@ -1,6 +1,7 @@
 """Pylons middleware initialization"""
 import urllib
 import logging 
+import json
 
 from beaker.middleware import CacheMiddleware, SessionMiddleware
 from paste.cascade import Cascade
@@ -101,10 +102,11 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 who_parser.remote_user_key,
            )
 
-    app = I18nMiddleware(app, config)
     # Establish the Registry for this application
     app = RegistryManager(app)
 
+    app = I18nMiddleware(app, config)
+
     if asbool(static_files):
         # Serve static files
         static_max_age = None if not asbool(config.get('ckan.cache_enabled')) \
@@ -124,6 +126,10 @@ def make_app(global_conf, full_stack=True, static_files=True, **app_conf):
                 )
         app = Cascade(extra_static_parsers+static_parsers)
 
+    # Page cache
+    if asbool(config.get('ckan.page_cache_enabled')):
+        app = PageCacheMiddleware(app, config)
+
     return app
 
 class I18nMiddleware(object):
@@ -134,17 +140,6 @@ def __init__(self, app, config):
         self.default_locale = config.get('ckan.locale_default', 'en')
         self.local_list = get_locales()
 
-    def get_cookie_lang(self, environ):
-        # get the lang from cookie if present
-        cookie = environ.get('HTTP_COOKIE')
-        if cookie:
-            cookies = [c.strip() for c in cookie.split(';')]
-            lang = [c.split('=')[1] for c in cookies \
-                    if c.startswith('ckan_lang')]
-            if lang and lang[0] in self.local_list:
-                return lang[0]
-        return None
-
     def __call__(self, environ, start_response):
         # strip the language selector from the requested url
         # and set environ variables for the language selected
@@ -165,16 +160,8 @@ def __call__(self, environ, start_response):
                 else:
                     environ['PATH_INFO'] = '/'
             else:
-                # use cookie lang or default language from config
-                cookie_lang = self.get_cookie_lang(environ)
-                if cookie_lang:
-                    environ['CKAN_LANG'] = cookie_lang
-                    default = (cookie_lang == self.default_locale)
-                    environ['CKAN_LANG_IS_DEFAULT'] = default
-                else:
-                    environ['CKAN_LANG'] = self.default_locale
-                    environ['CKAN_LANG_IS_DEFAULT'] = True
-
+                environ['CKAN_LANG'] = self.default_locale
+                environ['CKAN_LANG_IS_DEFAULT'] = True
 
             # Current application url
             path_info = environ['PATH_INFO']
@@ -191,3 +178,102 @@ def __call__(self, environ, start_response):
                 environ['CKAN_CURRENT_URL'] = path_info
 
         return self.app(environ, start_response)
+
+
+class PageCacheMiddleware(object):
+    ''' A simple page cache that can store and serve pages. It uses
+    Redis as storage. It caches pages that have a http status code of
+    200, use the GET method. Only non-logged in users receive cached
+    pages.
+    Cachable pages are indicated by a environ CKAN_PAGE_CACHABLE
+    variable.'''
+
+    def __init__(self, app, config):
+        self.app = app
+        import redis    # only import if used
+        self.redis = redis  # we need to reference this within the class
+        self.redis_exception = redis.exceptions.ConnectionError
+        self.redis_connection = None
+
+    def __call__(self, environ, start_response):
+
+        def _start_response(status, response_headers, exc_info=None):
+            # This wrapper allows us to get the status and headers.
+            environ['CKAN_PAGE_STATUS'] = status
+            environ['CKAN_PAGE_HEADERS'] = response_headers
+            return start_response(status, response_headers, exc_info)
+
+        # Only use cache for GET requests
+        # REMOTE_USER is used by some tests.
+        if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'):
+            return self.app(environ, start_response)
+
+        # If there is a ckan cookie (or auth_tkt) we avoid the cache.
+        # We want to allow other cookies like google analytics ones :(
+        cookie_string = environ.get('HTTP_COOKIE')
+        if cookie_string:
+            for cookie in cookie_string.split(';'):
+                if cookie.startswith('ckan') or cookie.startswith('auth_tkt'):
+                    return self.app(environ, start_response)
+
+        # Make our cache key
+        key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING'])
+
+        # Try to connect if we don't have a connection. Doing this here
+        # allows the redis server to be unavailable at times.
+        if self.redis_connection is None:
+            try:
+                self.redis_connection = self.redis.StrictRedis()
+                self.redis_connection.flushdb()
+            except self.redis_exception:
+                # Connection may have failed at flush so clear it.
+                self.redis_connection = None
+                return self.app(environ, start_response)
+
+        # If cached return cached result
+        try:
+            result = self.redis_connection.lrange(key, 0, 2)
+        except self.redis_exception:
+            # Connection failed so clear it and return the page as normal.
+            self.redis_connection = None
+            return self.app(environ, start_response)
+
+        if result:
+            headers = json.loads(result[1])
+            # Convert headers from list to tuples.
+            headers = [(str(key), str(value)) for key, value in headers]
+            start_response(str(result[0]), headers)
+            # Returning a huge string slows down the server. Therefore we
+            # cut it up into more usable chunks.
+            page = result[2]
+            out = []
+            total = len(page)
+            position = 0
+            size = 4096
+            while position < total:
+                out.append(page[position:position + size])
+                position += size
+            return out
+
+        # Generate the response from our application.
+        page = self.app(environ, _start_response)
+
+        # Only cache http status 200 pages
+        if not environ['CKAN_PAGE_STATUS'].startswith('200'):
+            return page
+
+        cachable = False
+        if environ.get('CKAN_PAGE_CACHABLE'):
+            cachable = True
+
+        # Cache things if cachable.
+        if cachable:
+            # Make sure we consume any file handles etc.
+            page_string = ''.join(list(page))
+            # Use a pipe to add page in a transaction.
+            pipe = self.redis_connection.pipeline()
+            pipe.rpush(key, environ['CKAN_PAGE_STATUS'])
+            pipe.rpush(key, json.dumps(environ['CKAN_PAGE_HEADERS']))
+            pipe.rpush(key, page_string)
+            pipe.execute()
+        return page