Merge branch 'master' into 792-create-dataset-auth-improvement

Conflicts:
	ckan/logic/auth/create.py
	ckan/tests/test_coding_standards.py

    trivial

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "doc/_themes/sphinx-theme-okfn"]
 	path = doc/_themes/sphinx-theme-okfn
-	url = git://github.com/okfn/sphinx-theme-okfn.git
+	url = https://github.com/okfn/sphinx-theme-okfn.git

CHANGELOG.rst

@@ -1,10 +1,13 @@
-.. _changelog:
+.. This tocdepth stops Sphinx from putting every subsection title in this file
+   into the master table of contents.
+
+:tocdepth: 1
 
 ---------
 Changelog
 ---------
 
-v2.0.1
+v2.0.1 2013-06-11
 =================
 
 Bug fixes:
@@ -420,7 +423,7 @@ v1.5 2011-11-07
 Major:
  * New visual theme (#1108)
     * Package & Resource edit overhaul (#1294/#1348/#1351/#1368/#1296)
-    * JS and CSS reorganisation (#1282, #1349, #1380)
+    * JS and CSS reorganization (#1282, #1349, #1380)
  * Apache Solr used for search in core instead of Postgres (#1275, #1361, #1365)
  * Authorization system now embedded in the logic layer (#1253)
  * Captcha added for user registration (#1307, #1431)

bin/ckan_edit_local.py

@@ -84,6 +84,7 @@ def canada_extras():
         'Level of Government':'level_of_government',
         }
     license_mapping = {
+        # CS: bad_spelling ignore
         'http://geogratis.ca/geogratis/en/licence.jsp':'geogratis',
         'Crown Copyright':'canada-crown',
         }

ckan/config/deployment.ini_tmpl

@@ -107,7 +107,7 @@ ckan.preview.loadable = html htm rdf+xml owl+xml xml n3 n-triples turtle plain a
 ckan.locale_default = en
 ckan.locale_order = en pt_BR ja it cs_CZ ca es fr el sv sr sr@latin no sk fi ru de pl nl bg ko_KR hu sa sl lv
 ckan.locales_offered =
-ckan.locales_filtered_out =
+ckan.locales_filtered_out = en_GB
 
 
 ## Feeds Settings

ckan/controllers/api.py

@@ -158,7 +158,7 @@ def action(self, logic_function, ver=None):
         except KeyError:
             log.error('Can\'t find logic function: %s' % logic_function)
             return self._finish_bad_request(
-                _('Action name not known: %s') % str(logic_function))
+                _('Action name not known: %s') % logic_function)
 
         context = {'model': model, 'session': model.Session, 'user': c.user,
                    'api_version': ver}
@@ -169,9 +169,9 @@ def action(self, logic_function, ver=None):
             request_data = self._get_request_data(try_url_params=
                                                   side_effect_free)
         except ValueError, inst:
-            log.error('Bad request data: %s' % str(inst))
+            log.error('Bad request data: %s' % inst)
             return self._finish_bad_request(
-                _('JSON Error: %s') % str(inst))
+                _('JSON Error: %s') % inst)
         if not isinstance(request_data, dict):
             # this occurs if request_data is blank
             log.error('Bad request data - not dict: %r' % request_data)
@@ -188,9 +188,11 @@ def action(self, logic_function, ver=None):
             return_dict['result'] = result
         except DataError, e:
             log.error('Format incorrect: %s - %s' % (e.error, request_data))
-            #TODO make better error message
-            return self._finish(400, _(u'Integrity Error') +
-                                ': %s - %s' % (e.error, request_data))
+            return_dict['error'] = {'__type': 'Integrity Error',
+                                    'message': e.error,
+                                    'data': request_data}
+            return_dict['success'] = False
+            return self._finish(400, return_dict, content_type='json')
         except NotAuthorized:
             return_dict['error'] = {'__type': 'Authorization Error',
                                     'message': _('Access denied')}
@@ -208,15 +210,9 @@ def action(self, logic_function, ver=None):
             error_dict['__type'] = 'Validation Error'
             return_dict['error'] = error_dict
             return_dict['success'] = False
+            # CS nasty_string ignore
             log.error('Validation error: %r' % str(e.error_dict))
             return self._finish(409, return_dict, content_type='json')
-        except logic.ParameterError, e:
-            return_dict['error'] = {'__type': 'Parameter Error',
-                                    'message': '%s: %s' %
-                                    (_('Parameter Error'), e.extra_msg)}
-            return_dict['success'] = False
-            log.error('Parameter error: %r' % e.extra_msg)
-            return self._finish(409, return_dict, content_type='json')
         except search.SearchQueryError, e:
             return_dict['error'] = {'__type': 'Search Query Error',
                                     'message': 'Search Query is invalid: %r' %
@@ -228,6 +224,12 @@ def action(self, logic_function, ver=None):
                                     'message': 'Search error: %r' % e.args}
             return_dict['success'] = False
             return self._finish(409, return_dict, content_type='json')
+        except search.SearchIndexError, e:
+            return_dict['error'] = {'__type': 'Search Index Error',
+                    'message': 'Unable to add package to search index: %s' %
+                    str(e)}
+            return_dict['success'] = False
+            return self._finish(500, return_dict, content_type='json')
         return self._finish_ok(return_dict)
 
     def _get_action_from_map(self, action_map, register, subregister):
@@ -333,7 +335,7 @@ def create(self, ver=None, register=None, subregister=None,
             data_dict.update(request_data)
         except ValueError, inst:
             return self._finish_bad_request(
-                _('JSON Error: %s') % str(inst))
+                _('JSON Error: %s') % inst)
 
         action = self._get_action_from_map(action_map, register, subregister)
         if not action:
@@ -356,13 +358,17 @@ def create(self, ver=None, register=None, subregister=None,
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
         except ValidationError, e:
+            # CS: nasty_string ignore
             log.error('Validation error: %r' % str(e.error_dict))
             return self._finish(409, e.error_dict, content_type='json')
         except DataError, e:
             log.error('Format incorrect: %s - %s' % (e.error, request_data))
-            #TODO make better error message
-            return self._finish(400, _(u'Integrity Error') +
-                                ': %s - %s' % (e.error, request_data))
+            error_dict = {
+                'success': False,
+                'error': {'__type': 'Integrity Error',
+                                    'message': e.error,
+                                    'data': request_data}}
+            return self._finish(400, error_dict, content_type='json')
         except search.SearchIndexError:
             log.error('Unable to add package to search index: %s' %
                       request_data)
@@ -392,7 +398,7 @@ def update(self, ver=None, register=None, subregister=None,
             data_dict.update(request_data)
         except ValueError, inst:
             return self._finish_bad_request(
-                _('JSON Error: %s') % str(inst))
+                _('JSON Error: %s') % inst)
 
         action = self._get_action_from_map(action_map, register, subregister)
         if not action:
@@ -408,13 +414,17 @@ def update(self, ver=None, register=None, subregister=None,
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
         except ValidationError, e:
+            # CS: nasty_string ignore
             log.error('Validation error: %r' % str(e.error_dict))
             return self._finish(409, e.error_dict, content_type='json')
         except DataError, e:
             log.error('Format incorrect: %s - %s' % (e.error, request_data))
-            #TODO make better error message
-            return self._finish(400, _(u'Integrity Error') +
-                                ': %s - %s' % (e.error, request_data))
+            error_dict = {
+                'success': False,
+                'error': {'__type': 'Integrity Error',
+                                    'message': e.error,
+                                    'data': request_data}}
+            return self._finish(400, error_dict, content_type='json')
         except search.SearchIndexError:
             log.error('Unable to update search index: %s' % request_data)
             return self._finish(500, _(u'Unable to update search index') %
@@ -452,6 +462,7 @@ def delete(self, ver=None, register=None, subregister=None,
             extra_msg = e.extra_msg
             return self._finish_not_found(extra_msg)
         except ValidationError, e:
+            # CS: nasty_string ignore
             log.error('Validation error: %r' % str(e.error_dict))
             return self._finish(409, e.error_dict, content_type='json')
 

ckan/controllers/group.py

@@ -524,7 +524,7 @@ def _save_edit(self, id, context):
             if id != group['name']:
                 self._force_reindex(group)
 
-            h.redirect_to('%s_read' % str(group['type']), id=group['name'])
+            h.redirect_to('%s_read' % group['type'], id=group['name'])
         except NotAuthorized:
             abort(401, _('Unauthorized to read group %s') % id)
         except NotFound, e:

ckan/controllers/package.py

@@ -75,9 +75,6 @@ def _new_template(self, package_type):
     def _edit_template(self, package_type):
         return lookup_package_plugin(package_type).edit_template()
 
-    def _comments_template(self, package_type):
-        return lookup_package_plugin(package_type).comments_template()
-
     def _search_template(self, package_type):
         return lookup_package_plugin(package_type).search_template()
 
@@ -357,27 +354,6 @@ def read(self, id, format='html'):
 
         return render(template, loader_class=loader)
 
-    def comments(self, id):
-        package_type = self._get_package_type(id)
-        context = {'model': model, 'session': model.Session,
-                   'user': c.user or c.author}
-
-        # check if package exists
-        try:
-            c.pkg_dict = get_action('package_show')(context, {'id': id})
-            c.pkg = context['package']
-        except NotFound:
-            abort(404, _('Dataset not found'))
-        except NotAuthorized:
-            abort(401, _('Unauthorized to read package %s') % id)
-
-        # used by disqus plugin
-        c.current_package_id = c.pkg.id
-
-        # render the package
-        package_saver.PackageSaver().render_package(c.pkg_dict)
-        return render(self._comments_template(package_type))
-
     def history(self, id):
         package_type = self._get_package_type(id.split('@')[0])
 
@@ -611,6 +587,9 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
                     data_dict = get_action('package_show')(context, {'id': id})
                 except NotAuthorized:
                     abort(401, _('Unauthorized to update dataset'))
+                except NotFound:
+                    abort(404,
+                      _('The dataset {id} could not be found.').format(id=id))
                 if not len(data_dict['resources']):
                     # no data so keep on page
                     msg = _('You must add at least one data resource')
@@ -640,6 +619,9 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
                 return self.new_resource(id, data, errors, error_summary)
             except NotAuthorized:
                 abort(401, _('Unauthorized to create a resource'))
+            except NotFound:
+                abort(404,
+                    _('The dataset {id} could not be found.').format(id=id))
             if save_action == 'go-metadata':
                 # go to final stage of add dataset
                 redirect(h.url_for(controller='package',
@@ -664,7 +646,10 @@ def new_resource(self, id, data=None, errors=None, error_summary=None):
         # get resources for sidebar
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author}
-        pkg_dict = get_action('package_show')(context, {'id': id})
+        try:
+            pkg_dict = get_action('package_show')(context, {'id': id})
+        except NotFound:
+            abort(404, _('The dataset {id} could not be found.').format(id=id))
         # required for nav menu
         vars['pkg_dict'] = pkg_dict
         if pkg_dict['state'] == 'draft':

ckan/lib/activity_streams.py

@@ -256,7 +256,7 @@ def activity_list_to_html(context, activity_stream, extra_vars):
 
         if not activity_type in activity_stream_string_functions:
             raise NotImplementedError("No activity renderer for activity "
-                "type '%s'" % str(activity_type))
+                "type '%s'" % activity_type)
 
         if activity_type in activity_stream_string_icons:
             activity_icon = activity_stream_string_icons[activity_type]

ckan/lib/base.py

@@ -263,6 +263,12 @@ def _identify_user(self):
         if not c.user:
             self._identify_user_default()
 
+        # If we have a user but not the userobj let's get the userobj.  This
+        # means that IAuthenticator extensions do not need to access the user
+        # model directly.
+        if c.user and not c.userobj:
+            c.userobj = model.User.by_name(c.user)
+
         # general settings
         if c.user:
             c.author = c.user

ckan/lib/cli.py

@@ -1,5 +1,6 @@
 import collections
 import csv
+import multiprocessing as mp
 import os
 import datetime
 import sys
@@ -8,6 +9,7 @@
 import ckan.include.rjsmin as rjsmin
 import ckan.include.rcssmin as rcssmin
 import ckan.lib.fanstatic_resources as fanstatic_resources
+import sqlalchemy as sa
 
 import paste.script
 from paste.registry import Registry
@@ -69,7 +71,7 @@ class CkanCommand(paste.script.command.Command):
     default_verbosity = 1
     group_name = 'ckan'
 
-    def _load_config(self):
+    def _get_config(self):
         from paste.deploy import appconfig
         if not self.options.config:
             msg = 'No config file supplied'
@@ -78,7 +80,10 @@ def _load_config(self):
         if not os.path.exists(self.filename):
             raise AssertionError('Config filename %r does not exist.' % self.filename)
         fileConfig(self.filename)
-        conf = appconfig('config:' + self.filename)
+        return appconfig('config:' + self.filename)
+
+    def _load_config(self):
+        conf = self._get_config()
         assert 'ckan' not in dir() # otherwise loggers would be disabled
         # We have now loaded the config. Now we can import ckan for the
         # first time.
@@ -308,12 +313,15 @@ def version(self):
         print Session.execute('select version from migrate_version;').fetchall()
 
 
+
 class SearchIndexCommand(CkanCommand):
     '''Creates a search index for all datasets
 
     Usage:
       search-index [-i] [-o] [-r] [-e] rebuild [dataset_name]  - reindex dataset_name if given, if not then rebuild
                                                                  full search index (all datasets)
+      search-index rebuild_fast                                - reindex using multiprocessing using all cores. 
+                                                                 This acts in the same way as rubuild -r [EXPERIMENTAL]
       search-index check                                       - checks for datasets not indexed
       search-index show DATASET_NAME                           - shows index of a dataset
       search-index clear [dataset_name]                        - clears the search index for the provided dataset or
@@ -346,14 +354,18 @@ def __init__(self,name):
                     )
 
     def command(self):
-        self._load_config()
-
         if not self.args:
             # default to printing help
             print self.usage
             return
 
         cmd = self.args[0]
+        # Do not run load_config yet
+        if cmd == 'rebuild_fast':
+            self.rebuild_fast()
+            return
+
+        self._load_config()
         if cmd == 'rebuild':
             self.rebuild()
         elif cmd == 'check':
@@ -402,6 +414,44 @@ def clear(self):
         package_id =self.args[1] if len(self.args) > 1 else None
         clear(package_id)
 
+    def rebuild_fast(self):
+        ###  Get out config but without starting pylons environment ####
+        conf = self._get_config()
+
+        ### Get ids using own engine, otherwise multiprocess will balk
+        db_url = conf['sqlalchemy.url']
+        engine = sa.create_engine(db_url)
+        package_ids = []
+        result = engine.execute("select id from package where state = 'active';")
+        for row in result:
+            package_ids.append(row[0])
+
+        def start(ids):
+            ## load actual enviroment for each subprocess, so each have thier own
+            ## sa session
+            self._load_config()
+            from ckan.lib.search import rebuild, commit
+            rebuild(package_ids=ids)
+            commit()
+
+        def chunks(l, n):
+            """ Yield n successive chunks from l.
+            """
+            newn = int(len(l) / n)
+            for i in xrange(0, n-1):
+                yield l[i*newn:i*newn+newn]
+            yield l[n*newn-newn:]
+
+        processes = []
+        for chunk in chunks(package_ids, mp.cpu_count()):
+            process = mp.Process(target=start, args=(chunk,))
+            processes.append(process)
+            process.daemon = True
+            process.start()
+
+        for process in processes:
+            process.join()
+
 class Notification(CkanCommand):
     '''Send out modification notifications.