Merge remote-tracking branch 'origin/master'

ckan/config/middleware.py

@@ -204,12 +204,18 @@ def _start_response(status, response_headers, exc_info=None):
             return start_response(status, response_headers, exc_info)
 
         # Only use cache for GET requests
-        # If there is a cookie we avoid the cache.
         # REMOTE_USER is used by some tests.
-        if environ['REQUEST_METHOD'] != 'GET' or environ.get('HTTP_COOKIE') or \
-           environ.get('REMOTE_USER'):
+        if environ['REQUEST_METHOD'] != 'GET' or environ.get('REMOTE_USER'):
             return self.app(environ, start_response)
 
+        # If there is a ckan cookie (or auth_tkt) we avoid the cache.
+        # We want to allow other cookies like google analytics ones :(
+        cookie_string = environ.get('HTTP_COOKIE')
+        if cookie_string:
+            for cookie in cookie_string.split(';'):
+                if cookie.startswith('ckan') or cookie.startswith('auth_tkt'):
+                    return self.app(environ, start_response)
+
         # Make our cache key
         key = 'page:%s?%s' % (environ['PATH_INFO'], environ['QUERY_STRING'])
 
@@ -220,6 +226,8 @@ def _start_response(status, response_headers, exc_info=None):
                 self.redis_connection = self.redis.StrictRedis()
                 self.redis_connection.flushdb()
             except self.redis_exception:
+                # Connection may have failed at flush so clear it.
+                self.redis_connection = None
                 return self.app(environ, start_response)
 
         # If cached return cached result

ckan/config/routing.py

@@ -25,7 +25,7 @@ def make_map():
     DELETE = dict(method=['DELETE'])
     GET_POST = dict(method=['GET', 'POST'])
     PUT_POST = dict(method=['PUT','POST'])
-    GET_POST_DELETE = dict(method=['GET', 'POST', 'DELETE'])
+    PUT_POST_DELETE = dict(method=['PUT', 'POST', 'DELETE'])
     OPTIONS = dict(method=['OPTIONS'])
 
     from ckan.lib.plugins import register_package_plugins
@@ -139,7 +139,7 @@ def make_map():
             m.connect('datastore_read', '/api/data/{id}{url:(/.*)?}',
                       action='read', url='', conditions=GET)
             m.connect('datastore_write', '/api/data/{id}{url:(/.*)?}',
-                      action='write', url='', conditions=GET_POST_DELETE)
+                      action='write', url='', conditions=PUT_POST_DELETE)
 
 
     map.redirect('/packages', '/dataset')

ckan/lib/base.py

@@ -180,17 +180,24 @@ def __call__(self, environ, start_response):
         # the request is routed to. This routing information is
         # available in environ['pylons.routes_dict']
 
-        # clean out any old cookies as they may contain api keys etc
+        # Clean out any old cookies as they may contain api keys etc
+        # This also improves the cachability of our pages as cookies
+        # prevent proxy servers from caching content unless they have
+        # been configured to ignore them.
         for cookie in request.cookies:
-            if cookie.startswith('ckan') and cookie not in ['ckan', 'ckan_killtopbar']:
+            if cookie.startswith('ckan') and cookie not in ['ckan']:
                 response.delete_cookie(cookie)
-
-            if cookie == 'ckan' and not c.user and not h.are_there_flash_messages():
+            # Remove the ckan session cookie if not used e.g. logged out
+            elif cookie == 'ckan' and not c.user and not h.are_there_flash_messages():
                 if session.id:
                     if not session.get('lang'):
                         session.delete()
                 else:
                     response.delete_cookie(cookie)
+            # Remove auth_tkt repoze.who cookie if user not logged in.
+            elif cookie == 'auth_tkt' and not session.id:
+                response.delete_cookie(cookie)
+
         try:
             return WSGIController.__call__(self, environ, start_response)
         finally:

ckan/lib/cli.py

@@ -8,12 +8,12 @@
 from paste.script.util.logging_config import fileConfig
 import re
 
-class MockTranslator(object): 
-    def gettext(self, value): 
-        return value 
+class MockTranslator(object):
+    def gettext(self, value):
+        return value
 
-    def ugettext(self, value): 
-        return value 
+    def ugettext(self, value):
+        return value
 
     def ungettext(self, singular, plural, n):
         if n > 1:
@@ -54,13 +54,13 @@ def _load_config(self):
         self.registry.register(pylons.translator, self.translator_obj)
 
     def _setup_app(self):
-        cmd = paste.script.appinstall.SetupCommand('setup-app') 
-        cmd.run([self.filename]) 
+        cmd = paste.script.appinstall.SetupCommand('setup-app')
+        cmd.run([self.filename])
 
 
 class ManageDb(CkanCommand):
     '''Perform various tasks on the database.
-    
+
     db create # alias of db upgrade
     db init # create and put in default data
     db clean
@@ -82,7 +82,7 @@ class ManageDb(CkanCommand):
     max_args = None
     min_args = 1
 
-    def command(self):        
+    def command(self):
         self._load_config()
         from ckan import model
         import ckan.lib.search as search
@@ -170,7 +170,7 @@ def _postgres_load(self, filepath):
         self._run_cmd(pg_cmd)
 
     def _run_cmd(self, command_line):
-        import subprocess    
+        import subprocess
         retcode = subprocess.call(command_line, shell=True)
         if retcode != 0:
             raise SystemError('Command exited with errorcode: %i' % retcode)
@@ -196,7 +196,7 @@ def load(self, only_load=False):
             print 'Upgrading DB'
             from ckan import model
             model.repo.upgrade_db()
-            
+
             print 'Rebuilding search index'
             import ckan.lib.search
             ckan.lib.search.rebuild()
@@ -270,47 +270,82 @@ class SearchIndexCommand(CkanCommand):
     '''Creates a search index for all datasets
 
     Usage:
-      search-index rebuild [package-name]  - reindex package-name if given, if not then rebuild full search index (all packages)
-      search-index check                   - checks for packages not indexed
-      search-index show {package-name}     - shows index of a package
-      search-index clear                   - clears the search index for this ckan instance
+      search-index [-i] [-o] [-r] rebuild [dataset-name]     - reindex dataset-name if given, if not then rebuild full search index (all datasets)
+      search-index check                                     - checks for datasets not indexed
+      search-index show {dataset-name}                       - shows index of a dataset
+      search-index clear [dataset-name]                      - clears the search index for the provided dataset or for the whole ckan instance
     '''
 
     summary = __doc__.split('\n')[0]
     usage = __doc__
     max_args = 2
     min_args = 0
 
+    def __init__(self,name):
+
+        super(SearchIndexCommand,self).__init__(name)
+
+        self.parser.add_option('-i', '--force', dest='force',
+            action='store_true', default=False, help='Ignore exceptions when rebuilding the index')
+
+        self.parser.add_option('-o', '--only-missing', dest='only_missing',
+            action='store_true', default=False, help='Index non indexed datasets only')
+
+        self.parser.add_option('-r', '--refresh', dest='refresh',
+            action='store_true', default=False, help='Refresh current index (does not clear the existing one)')
+
     def command(self):
         self._load_config()
-        from ckan.lib.search import rebuild, check, show, clear
 
         if not self.args:
             # default to printing help
             print self.usage
             return
 
-        cmd = self.args[0]        
+        cmd = self.args[0]
         if cmd == 'rebuild':
-            if len(self.args) > 1:
-                rebuild(self.args[1])
-            else:
-                rebuild()
+            self.rebuild()
         elif cmd == 'check':
-            check()
+            self.check()
         elif cmd == 'show':
-            if not len(self.args) == 2:
-                import pdb; pdb.set_trace()
-                self.args
-            show(self.args[1])
+            self.show()
         elif cmd == 'clear':
-            clear()
+            self.clear()
         else:
             print 'Command %s not recognized' % cmd
 
+    def rebuild(self):
+        from ckan.lib.search import rebuild
+
+        if len(self.args) > 1:
+            rebuild(self.args[1])
+        else:
+            rebuild(only_missing=self.options.only_missing,
+                    force=self.options.force,
+                    refresh=self.options.refresh)
+    def check(self):
+        from ckan.lib.search import check
+
+        check()
+
+    def show(self):
+        from ckan.lib.search import show
+
+        if not len(self.args) == 2:
+            print 'Missing parameter: dataset-name'
+            return
+        index = show(self.args[1])
+        pprint(index)
+
+    def clear(self):
+        from ckan.lib.search import clear
+
+        package_id =self.args[1] if len(self.args) > 1 else None
+        clear(package_id)
+
 class Notification(CkanCommand):
     '''Send out modification notifications.
-    
+
     In "replay" mode, an update signal is sent for each dataset in the database.
 
     Usage:
@@ -332,7 +367,7 @@ def command(self):
             cmd = 'replay'
         else:
             cmd = self.args[0]
-        
+
         if cmd == 'replay':
             dome = DomainObjectModificationExtension()
             for package in Session.query(Package):
@@ -466,12 +501,12 @@ def get_user_str(self, user):
         if user.name != user.display_name:
             user_str += ' display=%s' % user.display_name
         return user_str
-        
+
     def list(self):
         from ckan import model
         print 'Users:'
         users = model.Session.query(model.User)
-        print 'count = %i' % users.count()        
+        print 'count = %i' % users.count()
         for user in users:
             print self.get_user_str(user)
 
@@ -484,7 +519,7 @@ def show(self):
 
     def setpass(self):
         from ckan import model
-        
+
         if len(self.args) < 2:
             print 'Need name of the user.'
             return
@@ -524,7 +559,7 @@ def password_prompt(cls):
 
     def add(self):
         from ckan import model
-        
+
         if len(self.args) < 2:
             print 'Need name of the user.'
             return
@@ -561,10 +596,10 @@ def add(self):
 
         if not password:
             password = self.password_prompt()
-        
+
         print('Creating user: %r' % username)
 
-        
+
         user_params = {'name': unicode(username),
                        'password': password}
         if apikey:
@@ -641,7 +676,7 @@ def _get_dataset(self, dataset_ref):
         dataset = model.Package.get(unicode(dataset_ref))
         assert dataset, 'Could not find dataset matching reference: %r' % dataset_ref
         return dataset
-            
+
     def show(self, dataset_ref):
         from ckan import model
         import pprint
@@ -670,7 +705,7 @@ def purge(self, dataset_ref):
         dataset.purge()
         model.repo.commit_and_remove()
         print '%s purged' % name
-        
+
 
 class Celery(CkanCommand):
     '''Run celery daemon
@@ -686,7 +721,7 @@ def command(self):
         os.environ['CKAN_CONFIG'] = os.path.abspath(self.options.config)
         from ckan.lib.celery_app import celery
         celery.worker_main(argv=['celeryd', '--loglevel=INFO'])
-        
+
 
 class Ratings(CkanCommand):
     '''Manage the ratings stored in the db
@@ -721,7 +756,7 @@ def count(self):
         q = model.Session.query(model.Rating)
         print "%i ratings" % q.count()
         q = q.filter(model.Rating.user_id == None)
-        print "of which %i are anonymous ratings" % q.count()        
+        print "of which %i are anonymous ratings" % q.count()
 
     def clean(self, user_ratings=True):
         from ckan import model

ckan/lib/dictization/__init__.py

@@ -46,6 +46,11 @@ def table_dictize(obj, context, **kw):
 
     result_dict.update(kw)
 
+    ##HACK For optimisation to get metadata_modified created faster.
+
+    context['metadata_modified'] = max(result_dict.get('revision_timestamp', ''),
+                                       context.get('metadata_modified', ''))
+
     return result_dict
 
 

ckan/lib/dictization/model_dictize.py

@@ -40,9 +40,10 @@ def resource_list_dictize(res_list, context):
     active = context.get('active', True)
     result_list = []
     for res in res_list:
+        resource_dict = resource_dictize(res, context)
         if active and res.state not in ('active', 'pending'):
             continue
-        result_list.append(resource_dictize(res, context))
+        result_list.append(resource_dict)
 
     return sorted(result_list, key=lambda x: x["position"])
 
@@ -65,9 +66,9 @@ def extras_list_dictize(extras_list, context):
     result_list = []
     active = context.get('active', True)
     for extra in extras_list:
+        dictized = d.table_dictize(extra, context)
         if active and extra.state not in ('active', 'pending'):
             continue
-        dictized = d.table_dictize(extra, context)
         value = dictized["value"]
         if not(context.get("extras_as_string") and isinstance(value, basestring)):
             dictized["value"] = h.json.dumps(value)
@@ -205,11 +206,10 @@ def package_dictize(pkg, context):
         result_dict['license_title']= pkg.license_id
 
     # creation and modification date
-    result_dict['metadata_modified'] = pkg.metadata_modified.isoformat() \
-        if pkg.metadata_modified else None
+    result_dict['metadata_modified'] = context.pop('metadata_modified')
     result_dict['metadata_created'] = pkg.metadata_created.isoformat() \
         if pkg.metadata_created else None
-
+        
     if context.get('for_view'):
         for item in plugins.PluginImplementations(plugins.IPackageController):
             result_dict = item.before_view(result_dict)
@@ -366,14 +366,6 @@ def package_to_api(pkg, context):
     if site_url:
         dictized['ckan_url'] = '%s/dataset/%s' % (site_url, pkg.name)
 
-    metadata_modified = pkg.metadata_modified
-    dictized['metadata_modified'] = metadata_modified.isoformat() \
-        if metadata_modified else None
-
-    metadata_created = pkg.metadata_created
-    dictized['metadata_created'] = metadata_created.isoformat() \
-        if metadata_created else None
-
     for resource in dictized["resources"]:
         resource_dict_to_api(resource, pkg.id, context)