Merge branch 'master' of https://github.com/okfn/ckan

ckan/ckan_nose_plugin.py

@@ -1,10 +1,13 @@
 from nose.plugins import Plugin
 from inspect import isclass
+import hashlib
 import os
 import sys
+import re
 import pkg_resources
 from paste.deploy import loadapp
 from pylons import config
+import unittest
 import time
 
 class CkanNose(Plugin):
@@ -54,12 +57,35 @@ def options(self, parser, env):
             action='store_true',
             dest='docstrings',
             help='set this to display test docstrings instead of module names')
+        parser.add_option(
+            '--segments',
+            dest='segments',
+            help='A string containing a hex digits that represent which of'
+                 'the 16 test segments to run. i.e 15af will run segments 1,5,a,f')
+
+    def wantClass(self, cls):
+        name = cls.__name__
+
+        wanted = (not cls.__name__.startswith('_')
+                  and (issubclass(cls, unittest.TestCase)
+                       or re.search('(?:^|[\b_\./-])[Tt]est', name)
+                      ))
+
+        if self.segments and str(hashlib.md5(name).hexdigest())[0] not in self.segments:
+            return False
+
+        return wanted
+
+    def finalize(self, report):
+        if self.segments:
+            print 'Segments: %s' % self.segments
 
     def configure(self, settings, config):
         CkanNose.settings = settings
         if settings.is_ckan:
             self.enabled = True
             self.is_first_test = True
+        self.segments = settings.segments
 
     def describeTest(self, test):
         if not CkanNose.settings.docstrings:
@@ -70,22 +96,22 @@ def startTest(self, test):
         """
         startTest: start timing.
         """
-        self._started = time.time()
+##        self._started = time.time()
 
     def stopTest(self, test):
         """
         stopTest: stop timing, canonicalize the test name, and save
         the run time.
         """
-        runtime = time.time() - self._started
-
-        # CTB: HACK!
-        f = open('times.txt', 'a')
-
-        testname = str(test)
-        #if ' ' in testname:
-        #    testname = testname.split(' ')[1]
-
-        f.write('%s,%s\n' % (testname, str(runtime)))
-
-        f.close()
+##        runtime = time.time() - self._started
+##
+##        # CTB: HACK!
+##        f = open('times.txt', 'a')
+##
+##        testname = str(test)
+##        #if ' ' in testname:
+##        #    testname = testname.split(' ')[1]
+##
+##        f.write('%s,%s\n' % (testname, str(runtime)))
+##
+##        f.close()

ckan/config/environment.py

@@ -75,6 +75,10 @@ def find_controller(self, controller):
         plugin.update_config(config)
 
     # This is set up before globals are initialized
+    site_id = os.environ.get('CKAN_SITE_ID')
+    if site_id:
+        config['ckan.site_id'] = site_id
+
     site_url = config.get('ckan.site_url', '')
     ckan_host = config['ckan.host'] = urlparse(site_url).netloc
     if config.get('ckan.site_id') is None:
@@ -125,7 +129,12 @@ def template_loaded(template):
     warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning)
     warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning)
 
-    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
+    ckan_db = os.environ.get('CKAN_DB') 
+
+    if ckan_db:
+        engine = sqlalchemy.create_engine(ckan_db)
+    else:
+        engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
 
     if not model.meta.engine:
         model.init_model(engine)

ckan/config/routing.py

@@ -27,8 +27,8 @@ def make_map():
     PUT_POST = dict(method=['PUT','POST'])
     OPTIONS = dict(method=['OPTIONS'])
 
-    from lib.plugins import register_package_plugins
-    from lib.plugins import register_group_plugins
+    from ckan.lib.plugins import register_package_plugins
+    from ckan.lib.plugins import register_group_plugins
 
     map = Mapper(directory=config['pylons.paths']['controllers'],
                  always_scan=config['debug'])

ckan/controllers/group.py

@@ -14,7 +14,7 @@
 import ckan.forms
 import ckan.logic.action.get
 
-from lib.plugins import lookup_group_plugin
+from ckan.lib.plugins import lookup_group_plugin
 
 log = logging.getLogger(__name__)
 
@@ -233,10 +233,6 @@ def edit(self, id, data=None, errors=None, error_summary=None):
             old_data = get_action('group_show')(context, data_dict)
             c.grouptitle = old_data.get('title')
             c.groupname = old_data.get('name')
-            schema = self._db_to_form_schema()
-            if schema and not data:
-                old_data, errors = validate(old_data, schema, context=context)
-
             data = data or old_data
         except NotFound:
             abort(404, _('Group not found'))

ckan/controllers/package.py

@@ -23,7 +23,7 @@
 import ckan.logic.action.get
 from home import CACHE_PARAMETER
 
-from lib.plugins import lookup_package_plugin
+from ckan.lib.plugins import lookup_package_plugin
 
 log = logging.getLogger(__name__)
 
@@ -428,9 +428,6 @@ def edit(self, id, data=None, errors=None, error_summary=None):
             return self._save_edit(id, context)
         try:
             old_data = get_action('package_show')(context, {'id':id})
-            schema = self._db_to_form_schema(package_type=package_type)
-            if schema and not data:
-                old_data, errors = validate(old_data, schema, context=context)
             data = data or old_data
             # Merge all elements for the complete package dictionary
             c.pkg_dict = dict(old_data.items() + data.items())

ckan/lib/plugins.py

@@ -207,15 +207,6 @@ def history_template(self):
     def package_form(self):
         return 'package/new_package_form.html'
 
-    def form_to_db_schema(self):
-        schema =  logic.schema.package_form_schema()
-        schema['groups'] = {
-                'name': [not_empty, val.group_id_or_name_exists, unicode],
-                'id':   [ignore_missing, unicode],
-            }
-        return schema
-
-
     def form_to_db_schema_options(self, options):
         ''' This allows us to select different schemas for different
         purpose eg via the web interface or via the api or creation vs

ckan/logic/action/create.py

@@ -1,7 +1,7 @@
 import logging
 from pylons.i18n import _
 
-import lib.plugins as lib_plugins
+import ckan.lib.plugins as lib_plugins
 import ckan.logic as logic
 import ckan.rating as ratings
 import ckan.plugins as plugins

ckan/logic/action/get.py

@@ -15,13 +15,16 @@
 import ckan.logic as logic
 import ckan.logic.action
 import ckan.lib.dictization.model_dictize as model_dictize
+import ckan.lib.navl.dictization_functions
 import ckan.model.misc as misc
 import ckan.plugins as plugins
 import ckan.lib.search as search
+import ckan.lib.plugins as lib_plugins
 
 log = logging.getLogger('ckan.logic')
 
 # define some shortcuts
+validate = ckan.lib.navl.dictization_functions.validate
 table_dictize = ckan.lib.dictization.table_dictize
 render = ckan.lib.base.render
 Authorizer = ckan.authz.Authorizer
@@ -370,6 +373,7 @@ def package_relationships_list(context, data_dict):
 def package_show(context, data_dict):
 
     model = context['model']
+    context['session'] = model.Session
     name_or_id = data_dict.get("id") or data_dict['name_or_id']
 
     pkg = model.Package.get(name_or_id)
@@ -386,6 +390,11 @@ def package_show(context, data_dict):
     for item in plugins.PluginImplementations(plugins.IPackageController):
         item.read(pkg)
 
+    schema = lib_plugins.lookup_package_plugin(package_dict['type']).db_to_form_schema()
+
+    if schema:
+        package_dict, errors = validate(package_dict, schema, context=context)
+
     return package_dict
 
 def resource_show(context, data_dict):
@@ -436,6 +445,11 @@ def group_show(context, data_dict):
     for item in plugins.PluginImplementations(plugins.IGroupController):
         item.read(group)
 
+    schema = lib_plugins.lookup_group_plugin(group_dict['type']).db_to_form_schema()
+
+    if schema:
+        package_dict, errors = validate(group_dict, schema, context=context)
+
     return group_dict
 
 def group_package_show(context, data_dict):

ckan/logic/action/update.py

@@ -12,7 +12,7 @@
 import ckan.lib.dictization.model_save as model_save
 import ckan.lib.navl.dictization_functions
 import ckan.lib.navl.validators as validators
-import lib.plugins as lib_plugins
+import ckan.lib.plugins as lib_plugins
 
 log = logging.getLogger(__name__)
 
@@ -138,7 +138,7 @@ def package_update(context, data_dict):
 
     model = context['model']
     user = context['user']
-    name_or_id = data_dict.get("id") or data_dict['name_or_id']
+    name_or_id = data_dict.get("id") or data_dict['name']
     model.Session.remove()
     model.Session()._context = context
 

ckan/logic/schema.py

@@ -115,6 +115,7 @@ def default_package_schema():
         'url': [ignore_missing, unicode],#, URL(add_http=False)],
         'version': [ignore_missing, unicode, package_version_validator],
         'state': [ignore_not_package_admin, ignore_missing],
+        'type': [ignore_missing, unicode],
         '__extras': [ignore],
         '__junk': [empty],
         'resources': default_resource_schema(),
@@ -125,6 +126,7 @@ def default_package_schema():
         'groups': {
             'id': [ignore_missing, unicode],
             'name': [ignore_missing, unicode],
+            'title': [ignore_missing, unicode],
             '__extras': [ignore],
         }
     }
@@ -153,8 +155,7 @@ def package_form_schema():
     schema['log_message'] = [ignore_missing, unicode, no_http]
     schema['groups'] = {
             'id': [ignore_missing, unicode],
-            '__extras': [empty],
-            'name': [ignore, unicode],
+            '__extras': [ignore],
     }
     schema['tag_string'] = [ignore_missing, tag_string_convert]
     schema['extras_validation'] = [duplicate_extras_key, ignore]

ckan/logic/validators.py

@@ -312,6 +312,9 @@ def ignore_not_package_admin(key, data, errors, context):
     model = context['model']
     user = context.get('user')
 
+    if 'ignore_auth' in context:
+        return
+
     if user and Authorizer.is_sysadmin(user):
         return