Merge branch 'master' into 547-plugins-love

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "doc/_themes/sphinx-theme-okfn"]
 	path = doc/_themes/sphinx-theme-okfn
-	url = git://github.com/okfn/sphinx-theme-okfn.git
+	url = https://github.com/okfn/sphinx-theme-okfn.git

ckan/controllers/api.py

@@ -188,9 +188,11 @@ def action(self, logic_function, ver=None):
             return_dict['result'] = result
         except DataError, e:
             log.error('Format incorrect: %s - %s' % (e.error, request_data))
-            #TODO make better error message
-            return self._finish(400, _(u'Integrity Error') +
-                                ': %s - %s' % (e.error, request_data))
+            return_dict['error'] = {'__type': 'Integrity Error',
+                                    'message': e.error,
+                                    'data': request_data}
+            return_dict['success'] = False
+            return self._finish(400, return_dict, content_type='json')
         except NotAuthorized:
             return_dict['error'] = {'__type': 'Authorization Error',
                                     'message': _('Access denied')}
@@ -210,13 +212,6 @@ def action(self, logic_function, ver=None):
             return_dict['success'] = False
             log.error('Validation error: %r' % str(e.error_dict))
             return self._finish(409, return_dict, content_type='json')
-        except logic.ParameterError, e:
-            return_dict['error'] = {'__type': 'Parameter Error',
-                                    'message': '%s: %s' %
-                                    (_('Parameter Error'), e.extra_msg)}
-            return_dict['success'] = False
-            log.error('Parameter error: %r' % e.extra_msg)
-            return self._finish(409, return_dict, content_type='json')
         except search.SearchQueryError, e:
             return_dict['error'] = {'__type': 'Search Query Error',
                                     'message': 'Search Query is invalid: %r' %
@@ -366,9 +361,12 @@ def create(self, ver=None, register=None, subregister=None,
             return self._finish(409, e.error_dict, content_type='json')
         except DataError, e:
             log.error('Format incorrect: %s - %s' % (e.error, request_data))
-            #TODO make better error message
-            return self._finish(400, _(u'Integrity Error') +
-                                ': %s - %s' % (e.error, request_data))
+            error_dict = {
+                'success': False,
+                'error': {'__type': 'Integrity Error',
+                                    'message': e.error,
+                                    'data': request_data}}
+            return self._finish(400, error_dict, content_type='json')
         except search.SearchIndexError:
             log.error('Unable to add package to search index: %s' %
                       request_data)
@@ -418,9 +416,12 @@ def update(self, ver=None, register=None, subregister=None,
             return self._finish(409, e.error_dict, content_type='json')
         except DataError, e:
             log.error('Format incorrect: %s - %s' % (e.error, request_data))
-            #TODO make better error message
-            return self._finish(400, _(u'Integrity Error') +
-                                ': %s - %s' % (e.error, request_data))
+            error_dict = {
+                'success': False,
+                'error': {'__type': 'Integrity Error',
+                                    'message': e.error,
+                                    'data': request_data}}
+            return self._finish(400, error_dict, content_type='json')
         except search.SearchIndexError:
             log.error('Unable to update search index: %s' % request_data)
             return self._finish(500, _(u'Unable to update search index') %

ckan/controllers/package.py

@@ -75,9 +75,6 @@ def _new_template(self, package_type):
     def _edit_template(self, package_type):
         return lookup_package_plugin(package_type).edit_template()
 
-    def _comments_template(self, package_type):
-        return lookup_package_plugin(package_type).comments_template()
-
     def _search_template(self, package_type):
         return lookup_package_plugin(package_type).search_template()
 
@@ -357,27 +354,6 @@ def read(self, id, format='html'):
 
         return render(template, loader_class=loader)
 
-    def comments(self, id):
-        package_type = self._get_package_type(id)
-        context = {'model': model, 'session': model.Session,
-                   'user': c.user or c.author}
-
-        # check if package exists
-        try:
-            c.pkg_dict = get_action('package_show')(context, {'id': id})
-            c.pkg = context['package']
-        except NotFound:
-            abort(404, _('Dataset not found'))
-        except NotAuthorized:
-            abort(401, _('Unauthorized to read package %s') % id)
-
-        # used by disqus plugin
-        c.current_package_id = c.pkg.id
-
-        # render the package
-        package_saver.PackageSaver().render_package(c.pkg_dict)
-        return render(self._comments_template(package_type))
-
     def history(self, id):
         package_type = self._get_package_type(id.split('@')[0])
 
@@ -611,6 +587,9 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
                     data_dict = get_action('package_show')(context, {'id': id})
                 except NotAuthorized:
                     abort(401, _('Unauthorized to update dataset'))
+                except NotFound:
+                    abort(404,
+                      _('The dataset {id} could not be found.').format(id=id))
                 if not len(data_dict['resources']):
                     # no data so keep on page
                     msg = _('You must add at least one data resource')
@@ -640,6 +619,9 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
                 return self.new_resource(id, data, errors, error_summary)
             except NotAuthorized:
                 abort(401, _('Unauthorized to create a resource'))
+            except NotFound:
+                abort(404,
+                    _('The dataset {id} could not be found.').format(id=id))
             if save_action == 'go-metadata':
                 # go to final stage of add dataset
                 redirect(h.url_for(controller='package',
@@ -664,7 +646,10 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
         # get resources for sidebar
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author}
-        pkg_dict = get_action('package_show')(context, {'id': id})
+        try:
+            pkg_dict = get_action('package_show')(context, {'id': id})
+        except NotFound:
+            abort(404, _('The dataset {id} could not be found.').format(id=id))
         # required for nav menu
         vars['pkg_dict'] = pkg_dict
         if pkg_dict['state'] == 'draft':

ckan/lib/base.py

@@ -263,6 +263,12 @@ def _identify_user(self):
         if not c.user:
             self._identify_user_default()
 
+        # If we have a user but not the userobj let's get the userobj.  This
+        # means that IAuthenticator extensions do not need to access the user
+        # model directly.
+        if c.user and not c.userobj:
+            c.userobj = model.User.by_name(c.user)
+
         # general settings
         if c.user:
             c.author = c.user

ckan/lib/cli.py

@@ -1,5 +1,6 @@
 import collections
 import csv
+import multiprocessing as mp
 import os
 import datetime
 import sys
@@ -8,6 +9,7 @@
 import ckan.include.rjsmin as rjsmin
 import ckan.include.rcssmin as rcssmin
 import ckan.lib.fanstatic_resources as fanstatic_resources
+import sqlalchemy as sa
 
 import paste.script
 from paste.registry import Registry
@@ -69,7 +71,7 @@ class CkanCommand(paste.script.command.Command):
     default_verbosity = 1
     group_name = 'ckan'
 
-    def _load_config(self):
+    def _get_config(self):
         from paste.deploy import appconfig
         if not self.options.config:
             msg = 'No config file supplied'
@@ -78,7 +80,10 @@ def _load_config(self):
         if not os.path.exists(self.filename):
             raise AssertionError('Config filename %r does not exist.' % self.filename)
         fileConfig(self.filename)
-        conf = appconfig('config:' + self.filename)
+        return appconfig('config:' + self.filename)
+
+    def _load_config(self):
+        conf = self._get_config()
         assert 'ckan' not in dir() # otherwise loggers would be disabled
         # We have now loaded the config. Now we can import ckan for the
         # first time.
@@ -308,12 +313,15 @@ def version(self):
         print Session.execute('select version from migrate_version;').fetchall()
 
 
+
 class SearchIndexCommand(CkanCommand):
     '''Creates a search index for all datasets
 
     Usage:
       search-index [-i] [-o] [-r] [-e] rebuild [dataset_name]  - reindex dataset_name if given, if not then rebuild
                                                                  full search index (all datasets)
+      search-index rebuild_fast                                - reindex using multiprocessing using all cores. 
+                                                                 This acts in the same way as rubuild -r [EXPERIMENTAL]
       search-index check                                       - checks for datasets not indexed
       search-index show DATASET_NAME                           - shows index of a dataset
       search-index clear [dataset_name]                        - clears the search index for the provided dataset or
@@ -346,14 +354,18 @@ def __init__(self,name):
                     )
 
     def command(self):
-        self._load_config()
-
         if not self.args:
             # default to printing help
             print self.usage
             return
 
         cmd = self.args[0]
+        # Do not run load_config yet
+        if cmd == 'rebuild_fast':
+            self.rebuild_fast()
+            return
+
+        self._load_config()
         if cmd == 'rebuild':
             self.rebuild()
         elif cmd == 'check':
@@ -402,6 +414,44 @@ def clear(self):
         package_id =self.args[1] if len(self.args) > 1 else None
         clear(package_id)
 
+    def rebuild_fast(self):
+        ###  Get out config but without starting pylons environment ####
+        conf = self._get_config()
+
+        ### Get ids using own engine, otherwise multiprocess will balk
+        db_url = conf['sqlalchemy.url']
+        engine = sa.create_engine(db_url)
+        package_ids = []
+        result = engine.execute("select id from package where state = 'active';")
+        for row in result:
+            package_ids.append(row[0])
+
+        def start(ids):
+            ## load actual enviroment for each subprocess, so each have thier own
+            ## sa session
+            self._load_config()
+            from ckan.lib.search import rebuild, commit
+            rebuild(package_ids=ids)
+            commit()
+
+        def chunks(l, n):
+            """ Yield n successive chunks from l.
+            """
+            newn = int(len(l) / n)
+            for i in xrange(0, n-1):
+                yield l[i*newn:i*newn+newn]
+            yield l[n*newn-newn:]
+
+        processes = []
+        for chunk in chunks(package_ids, mp.cpu_count()):
+            process = mp.Process(target=start, args=(chunk,))
+            processes.append(process)
+            process.daemon = True
+            process.start()
+
+        for process in processes:
+            process.join()
+
 class Notification(CkanCommand):
     '''Send out modification notifications.
 

ckan/lib/email_notifications.py

@@ -31,7 +31,7 @@ def string_to_timedelta(s):
     7 days, 3:23:34.087465
     .087465 (microseconds only)
 
-    :raises ckan.logic.ParameterError: if the given string does not match any
+    :raises ckan.logic.ValidationError: if the given string does not match any
         of the recognised formats
 
     '''
@@ -56,7 +56,7 @@ def string_to_timedelta(s):
             break
 
     if not match:
-        raise logic.ParameterError('Not a valid time: {0}'.format(s))
+        raise logic.ValidationError('Not a valid time: {0}'.format(s))
 
     gd = match.groupdict()
     days = int(gd.get('days', '0'))

ckan/lib/helpers.py

@@ -86,9 +86,17 @@ def url_for_static(*args, **kw):
     '''Create url for static content that does not get translated
     eg css, js
     wrapper for routes.url_for'''
-    # make sure that if we specify the url that it is not unicode
+
+    def fix_arg(arg):
+        # make sure that if we specify the url that it is not unicode and
+        # starts with a /
+        arg = str(arg)
+        if not arg.startswith('/'):
+            arg = '/' + arg
+        return arg
+
     if args:
-        args = (str(args[0]), ) + args[1:]
+        args = (fix_arg(args[0]), ) + args[1:]
     my_url = _routes_default_url_for(*args, **kw)
     return my_url
 
@@ -1449,7 +1457,12 @@ def format_resource_items(items):
             continue
         # size is treated specially as we want to show in MiB etc
         if key == 'size':
-            value = formatters.localised_filesize(int(value))
+            try:
+                value = formatters.localised_filesize(int(value))
+            except ValueError:
+                # Sometimes values that can't be converted to ints can sneak
+                # into the db. In this case, just leave them as they are.
+                pass
         elif isinstance(value, basestring):
             # check if strings are actually datetime/number etc
             if re.search(reg_ex_datetime, value):

ckan/lib/navl/dictization_functions.py

@@ -117,7 +117,7 @@ def augment_data(data, schema):
 
     full_schema = make_full_schema(data, schema)
 
-    new_data = copy.deepcopy(data)
+    new_data = copy.copy(data)
 
     ## fill junk and extras
 

ckan/lib/plugins.py

@@ -230,9 +230,6 @@ def read_template(self):
     def edit_template(self):
         return 'package/edit.html'
 
-    def comments_template(self):
-        return 'package/comments.html'
-
     def search_template(self):
         return 'package/search.html'
 

ckan/lib/search/__init__.py

@@ -134,7 +134,7 @@ def notify(self, entity, operation):
             log.warn("Discarded Sync. indexing for: %s" % entity)
 
 
-def rebuild(package_id=None, only_missing=False, force=False, refresh=False, defer_commit=False):
+def rebuild(package_id=None, only_missing=False, force=False, refresh=False, defer_commit=False, package_ids=None):
     '''
         Rebuilds the search index.
 
@@ -155,6 +155,13 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def
         log.info('Indexing just package %r...', pkg_dict['name'])
         package_index.remove_dict(pkg_dict)
         package_index.insert_dict(pkg_dict)
+    elif package_ids:
+        for package_id in package_ids:
+            pkg_dict = logic.get_action('package_show')(
+                {'model': model, 'ignore_auth': True, 'validate': False},
+                {'id': package_id})
+            log.info('Indexing just package %r...', pkg_dict['name'])
+            package_index.update_dict(pkg_dict, True)
     else:
         package_ids = [r[0] for r in model.Session.query(model.Package.id).
                        filter(model.Package.state == 'active').all()]