Merge branch 'master' into enhancement-2220-cache-friendly

Conflicts:
	ckan/controllers/api.py
	ckan/controllers/package.py
	ckan/controllers/user.py
	ckan/lib/dictization/model_dictize.py
	ckan/lib/dictization/model_save.py

README.rst

@@ -9,15 +9,15 @@ http://ckan.org/.
  * Developer mailing list: ckan-dev@lists.okfn.org
  * Issue tracker: http://trac.ckan.org/
 
- Building Documentation
- ======================
+Building Documentation
+======================
 
-1. Install python-sphinx
+1. Install python-sphinx (>= 1.1)
 
 2. Initialize the theme submodule::
 
-   git submodule init
-   git submodule update
+    git submodule init
+    git submodule update
 
 3. Run the command to build the docs::
 

ckan/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '1.6.1a'
+__version__ = '1.6.1b'
 __description__ = 'Comprehensive Knowledge Archive Network (CKAN) Software'
 __long_description__ = \
 '''CKAN software provides a hub for datasets. The flagship site running CKAN 

ckan/ckan_nose_plugin.py

@@ -5,6 +5,7 @@
 import pkg_resources
 from paste.deploy import loadapp
 from pylons import config
+import time
 
 class CkanNose(Plugin):
     settings = None
@@ -65,3 +66,26 @@ def describeTest(self, test):
             # display module name instead of docstring
             return False
 
+    def startTest(self, test):
+        """
+        startTest: start timing.
+        """
+        self._started = time.time()
+
+    def stopTest(self, test):
+        """
+        stopTest: stop timing, canonicalize the test name, and save
+        the run time.
+        """
+        runtime = time.time() - self._started
+
+        # CTB: HACK!
+        f = open('times.txt', 'a')
+
+        testname = str(test)
+        #if ' ' in testname:
+        #    testname = testname.split(' ')[1]
+
+        f.write('%s,%s\n' % (testname, str(runtime)))
+
+        f.close()

ckan/config/deployment.ini_tmpl

@@ -111,6 +111,11 @@ ckan.site_url =
 ## Favicon (default is the CKAN software favicon)
 ckan.favicon = /images/icons/ckan.ico
 
+## The gravatar default to use.  This can be any of the pre-defined strings
+## as defined on http://en.gravatar.com/site/implement/images/ (e.g. "identicon"
+## or "mm").  Or it can be a url, e.g. "http://example.com/images/avatar.jpg"
+ckan.gravatar_default = identicon
+
 ## Solr support
 #solr_url = http://127.0.0.1:8983/solr
 

ckan/config/environment.py

@@ -7,9 +7,12 @@
 from paste.deploy.converters import asbool
 
 # Suppress benign warning 'Unbuilt egg for setuptools'
-warnings.simplefilter('ignore', UserWarning) 
+warnings.simplefilter('ignore', UserWarning)
+
+
 import pylons
-from sqlalchemy import engine_from_config
+import sqlalchemy
+
 from pylons import config
 from pylons.i18n.translation import ugettext
 from genshi.template import TemplateLoader
@@ -27,9 +30,9 @@ def load_environment(global_conf, app_conf):
     """Configure the Pylons environment via the ``pylons.config``
     object
     """
-    
+
     ######  Pylons monkey-patch
-    # this must be run at a time when the env is semi-setup, thus inlined here. 
+    # this must be run at a time when the env is semi-setup, thus inlined here.
     # Required by the deliverance plugin and iATI
     from pylons.wsgiapp import PylonsApp
     import pkg_resources
@@ -45,7 +48,7 @@ def find_controller(self, controller):
             return mycontroller
         return find_controller_generic(self, controller)
     PylonsApp.find_controller = find_controller
-    ###### END evil monkey-patch 
+    ###### END evil monkey-patch
 
     os.environ['CKAN_CONFIG'] = global_conf['__file__']
 
@@ -57,20 +60,20 @@ def find_controller(self, controller):
                  templates=[os.path.join(root, 'templates')])
 
     # Initialize config with the basic options
-    
+
     config.init_app(global_conf, app_conf, package='ckan', paths=paths)
-    
+
     # load all CKAN plugins
     plugins.load_all(config)
 
     from ckan.plugins import PluginImplementations
     from ckan.plugins.interfaces import IConfigurer
-    
+
     for plugin in PluginImplementations(IConfigurer):
         # must do update in place as this does not work:
         # config = plugin.update_config(config)
         plugin.update_config(config)
-    
+
     # This is set up before globals are initialized
     site_url = config.get('ckan.site_url', '')
     ckan_host = config['ckan.host'] = urlparse(site_url).netloc
@@ -91,7 +94,7 @@ def find_controller(self, controller):
     config['routes.map'] = make_map()
     config['pylons.app_globals'] = app_globals.Globals()
     config['pylons.h'] = ckan.lib.helpers
-        
+
     ## redo template setup to use genshi.search_path (so remove std template setup)
     template_paths = [paths['templates'][0]]
     extra_template_paths = config.get('extra_template_paths', '')
@@ -110,24 +113,26 @@ def template_loaded(template):
     logging.getLogger("MARKDOWN").setLevel(logging.getLogger().level)
 
     # Create the Genshi TemplateLoader
-    # config['pylons.app_globals'].genshi_loader = TemplateLoader(
-    #    paths['templates'], auto_reload=True)
-    # tmpl_options["genshi.loader_callback"] = template_loaded
     config['pylons.app_globals'].genshi_loader = TemplateLoader(
         template_paths, auto_reload=True, callback=template_loaded)
 
     # CONFIGURATION OPTIONS HERE (note: all config options will override
-    # any Pylons config options)    
+    # any Pylons config options)
 
     # Setup the SQLAlchemy database engine
-    engine = engine_from_config(config, 'sqlalchemy.')
+    # Suppress a couple of sqlalchemy warnings
+    warnings.filterwarnings('ignore', '^Unicode type received non-unicode bind param value', sqlalchemy.exc.SAWarning)
+    warnings.filterwarnings('ignore', "^Did not recognize type 'BIGINT' of column 'size'", sqlalchemy.exc.SAWarning)
+    warnings.filterwarnings('ignore', "^Did not recognize type 'tsvector' of column 'search_vector'", sqlalchemy.exc.SAWarning)
+
+    engine = sqlalchemy.engine_from_config(config, 'sqlalchemy.')
 
     if not model.meta.engine:
         model.init_model(engine)
-    
+
     from ckan.plugins import PluginImplementations
     from ckan.plugins.interfaces import IConfigurable
-    
+
     for plugin in PluginImplementations(IConfigurable):
         plugin.configure(config)
-    
+

ckan/config/routing.py

@@ -116,6 +116,8 @@ def make_map():
                   conditions=GET)
         m.connect('/util/resource/format_autocomplete',
                   action='format_autocomplete', conditions=GET)
+        m.connect('/util/resource/format_icon',
+                  action='format_icon', conditions=GET)
         m.connect('/util/authorizationgroup/autocomplete',
                   action='authorizationgroup_autocomplete')
         m.connect('/util/group/autocomplete', action='group_autocomplete')
@@ -171,12 +173,14 @@ def make_map():
         m.connect('/dataset/{action}/{id}',
           requirements=dict(action='|'.join([
           'edit',
+          'editresources',
           'authz',
           'history',
           'read_ajax',
           'history_ajax',
           ]))
           )
+        m.connect('/dataset/{id}.{format}', action='read')
         m.connect('/dataset/{id}', action='read')
         m.connect('/dataset/{id}/resource/{resource_id}', action='resource_read')
 
@@ -189,7 +193,7 @@ def make_map():
     ##map.connect('/group/edit/{id}', controller='group_formalchemy', action='edit')
 
     # These named routes are used for custom group forms which will use the
-    # names below based on the group.type (dataset_group is the default type)
+    # names below based on the group.type ('group' is the default type)
     with SubMapper(map, controller='group') as m:
         m.connect('group_index', '/group', action='index')
         m.connect('group_list', '/group/list', action='list')

ckan/config/solr/schema-1.3.xml

@@ -138,6 +138,11 @@
 
     <field name="indexed_ts" type="date" indexed="true" stored="true" default="NOW" multiValued="false"/>
 
+    <!-- Copy the title field into titleString, and treat as a string
+         (rather than text type).  This allows us to sort on the titleString -->
+    <field name="titleString" type="string" indexed="true" stored="false" />
+    <copyField source="title" dest="titleString"/>
+
     <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
     <dynamicField name="*" type="string" indexed="true"  stored="false"/>
 </fields>

ckan/controllers/api.py

@@ -621,11 +621,14 @@ def convert_to_dict(user):
     def is_slug_valid(self):
         slug = request.params.get('slug') or ''
         slugtype = request.params.get('type') or ''
+        disallowed = ['new', 'edit', 'search']
         if slugtype==u'package':
-            response_data = dict(valid=not bool(common.package_exists(slug)))
+            response_data = dict(valid=not bool(common.package_exists(slug)
+                                 or slug in disallowed ))
             return self._finish_ok(response_data)
         if slugtype==u'group':
-            response_data = dict(valid=not bool(common.group_exists(slug)))
+            response_data = dict(valid=not bool(common.group_exists(slug) or
+                                slug in disallowed ))
             return self._finish_ok(response_data)
         return self._finish_bad_request('Bad slug type: %s' % slugtype)
 
@@ -696,6 +699,14 @@ def munge_tag(self):
         munged_tag = munge.munge_tag(tag)
         return self._finish_ok(munged_tag)
 
+    def format_icon(self):
+        f = request.params.get('format')
+        out = {
+            'format' : f,
+            'icon'   : h.icon_url(h.format_icon(f))
+            }
+        return self._finish_ok(out)
+
     def status(self):
         context = {'model': model, 'session': model.Session}
         data_dict = {}

ckan/controllers/datastore.py

@@ -1,5 +1,7 @@
 from ckan.lib.base import BaseController, abort, _, c, response, request, g
 import ckan.model as model
+from ckan.lib.helpers import json
+from ckan.lib.jsonp import jsonpify
 from ckan.logic import get_action, check_access
 from ckan.logic import NotFound, NotAuthorized, ValidationError
 
@@ -11,26 +13,36 @@ def _make_redirect(self, id, url=''):
         # headers must be ascii strings
         response.headers['X-Accel-Redirect'] = str(redirect)
 
+    @jsonpify
     def read(self, id, url=''):
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author}
 
         try:
             resource = get_action('resource_show')(context, {'id': id})
+            if not resource.get('webstore_url', ''):
+                return {
+                    'error': 'DataStore is disabled for this resource'
+                    }
             self._make_redirect(id, url)
             return ''
         except NotFound:
             abort(404, _('Resource not found'))
         except NotAuthorized:
             abort(401, _('Unauthorized to read resource %s') % id)
 
+    @jsonpify
     def write(self, id, url):
         context = {'model': model, 'session': model.Session,
                    'user': c.user or c.author}
         try:
             resource = model.Resource.get(id)
             if not resource:
                 abort(404, _('Resource not found'))
+            if not resource.webstore_url:
+                return {
+                    'error': 'DataStore is disabled for this resource'
+                    }
             context["resource"] = resource
             check_access('resource_update', context, {'id': id})
             self._make_redirect(id, url)