Merge branch 'master' of github.com:okfn/ckan into feature-1515-activ…

…ity-streams

ckan/config/routing.py

@@ -278,6 +278,7 @@ def make_map():
     map.connect('/tag/{id}', controller='tag', action='read')
     # users
     map.redirect("/users/{url:.*}", "/user/{url}")
+    map.redirect("/user/", "/user")
     map.connect('/user/edit', controller='user', action='edit')
     # Note: openid users have slashes in their ids, so need the wildcard
     # in the route.

ckan/controllers/group.py

@@ -1,3 +1,4 @@
+import logging
 import genshi
 import datetime
 from urllib import urlencode
@@ -19,6 +20,7 @@
 import ckan.forms
 import ckan.logic.action.get
 
+log = logging.getLogger(__name__)
 
 # Mapping from group-type strings to IDatasetForm instances
 _controller_behaviour_for = dict()

ckan/controllers/package.py

@@ -31,7 +31,7 @@
 import ckan.misc
 import ckan.logic.action.get
 
-log = logging.getLogger('ckan.controllers')
+log = logging.getLogger(__name__)
 
 def search_url(params):
     url = h.url_for(controller='package', action='search')

ckan/controllers/storage.py

@@ -30,10 +30,8 @@
 log = getLogger(__name__)
 
 
-# pairtree_version0_1 file for identifying folders
 BUCKET = config.get('ckan.storage.bucket', 'default')
 key_prefix = config.get('ckan.storage.key_prefix', 'file/')
-storage_dir = config.get('ckan.storage.directory', '')
 
 _eq_re = re.compile(r"^(.*)(=[0-9]*)$")
 def fix_stupid_pylons_encoding(data):
@@ -50,30 +48,16 @@ def fix_stupid_pylons_encoding(data):
 
 
 def get_ofs():
+    """Return a configured instance of the appropriate OFS driver.
     """
-    Return a configured instance of the appropriate OFS driver, in all
-    cases here this will be the local file storage so we fix the implementation
-    to use pairtree.
-    """
-    return get_impl("pairtree")(storage_dir=storage_dir)
-
-
-pairtree_marker_done = False
-def create_pairtree_marker():
-    """
-    Make sure that the file pairtree_version0_1 is present in storage_dir
-    and if not then create it.
-    """
-    global pairtree_marker_done
-    if pairtree_marker_done or not storage_dir:
-        return
-
-    path = os.path.join(storage_dir, 'pairtree_version0_1')
-    if not os.path.exists( path ):
-        open(path, 'w').close() 
-
-    pairtree_marker_done = True
-
+    storage_backend = config['ofs.impl']
+    kw = {}
+    for k,v in config.items():
+        if not k.startswith('ofs.') or k == 'ofs.impl':
+            continue
+        kw[k[4:]] = v
+    ofs = get_impl(storage_backend)(**kw)
+    return ofs
 
 
 def authorize(method, bucket, key, user, ofs):
@@ -99,13 +83,6 @@ def authorize(method, bucket, key, user, ofs):
 class StorageController(BaseController):
     '''Upload to storage backend.
     '''
-    def __before__(self, action, **params):
-        super(StorageController, self).__before__(action, **params)
-        if not storage_dir:
-            abort(404)
-        else:
-            create_pairtree_marker()            
-
     @property
     def ofs(self):
         return get_ofs()
@@ -188,14 +165,6 @@ def file(self, label):
 
 
 class StorageAPIController(BaseController):
-
-    def __before__(self, action, **params):
-        super(StorageAPIController, self).__before__(action, **params)
-        if not storage_dir:
-            abort(404)
-        else:
-            create_pairtree_marker()
-
     @property
     def ofs(self):
         return get_ofs()
@@ -260,10 +229,16 @@ def set_metadata(self, label):
     @jsonpify
     def get_metadata(self, label):
         bucket = BUCKET
-        url = h.url_for('storage_file',
-                label=label,
-                qualified=True
-                )
+        storage_backend = config['ofs.impl']
+        if storage_backend in ['google', 's3']:
+            if not label.startswith("/"):
+                label = "/" + label
+            url = "https://%s/%s%s" % (self.ofs.conn.server_name(), bucket, label)
+        else:
+            url = h.url_for('storage_file',
+                    label=label,
+                    qualified=True
+                    )
         if not self.ofs.exists(bucket, label):
             abort(404)
         metadata = self.ofs.get_metadata(bucket, label)
@@ -331,7 +306,7 @@ def _get_remote_form_data(self, label):
         acl = 'public-read'
         fields = [ {
                 'name': self.ofs.conn.provider.metadata_prefix + 'uploaded-by',
-                'value': c.userobj.name
+                'value': c.userobj.id
                 }]
         conditions = [ '{"%s": "%s"}' % (x['name'], x['value']) for x in
                 fields ]
@@ -353,22 +328,30 @@ def _get_remote_form_data(self, label):
             )
         # HACK: fix up some broken stuff from boto
         # e.g. should not have content-length-range in list of fields!
+        storage_backend = config['ofs.impl']
         for idx,field in enumerate(data['fields']):
+            if storage_backend == 'google':
+                if field['name'] == 'AWSAccessKeyId':
+                    field['name'] = 'GoogleAccessId'
             if field['name'] == 'content-length-range':
                 del data['fields'][idx]
         return data
 
     def _get_form_data(self, label):
-        data = {
-            'action': h.url_for('storage_upload_handle', qualified=True),
-            'fields': [
-                {
-                    'name': 'key',
-                    'value': label
-                }
-            ]
-        }
-        return data
+        storage_backend = config['ofs.impl']
+        if storage_backend in ['google', 's3']:
+            return self._get_remote_form_data(label)
+        else:
+            data = {
+                'action': h.url_for('storage_upload_handle', qualified=True),
+                'fields': [
+                    {
+                        'name': 'key',
+                        'value': label
+                    }
+                ]
+            }
+            return data
 
     @jsonpify
     def auth_form(self, label):

ckan/lib/helpers.py

@@ -222,7 +222,7 @@ def group_name_to_title(name):
     return name
 
 def markdown_extract(text, extract_length=190):
-    if (text is None) or (text == ''):
+    if (text is None) or (text.strip() == ''):
         return ''
     html = fromstring(markdown(text))
     plain = html.xpath("string()")
@@ -310,7 +310,7 @@ def dataset_display_name(package_or_package_dict):
     if isinstance(package_or_package_dict, dict):
         return package_or_package_dict.get('title', '') or package_or_package_dict.get('name', '')
     else:
-        return package_or_package_dict.title or package_or_package_dictname
+        return package_or_package_dict.title or package_or_package_dict.name
 
 def dataset_link(package_or_package_dict):
     if isinstance(package_or_package_dict, dict):

ckan/lib/search/index.py

@@ -7,6 +7,8 @@
 
 from common import SearchIndexError, make_connection
 from ckan.model import PackageRelationship
+from ckan.plugins import (PluginImplementations,
+                          IPackageController)
 
 log = logging.getLogger(__name__)
 
@@ -138,6 +140,11 @@ def index_package(self, pkg_dict):
         import hashlib
         pkg_dict['index_id'] = hashlib.md5('%s%s' % (pkg_dict['id'],config.get('ckan.site_id'))).hexdigest()
 
+        for item in PluginImplementations(IPackageController):
+            pkg_dict = item.before_index(pkg_dict)
+
+        assert pkg_dict, 'Plugin must return non empty package dict on index'
+
         # send to solr:  
         try:
             conn.add_many([pkg_dict])

ckan/logic/action/create.py

@@ -72,6 +72,7 @@ def package_create(context, data_dict):
     model.setup_default_user_roles(pkg, admins)
     # Needed to let extensions know the package id
     model.Session.flush()
+
     for item in PluginImplementations(IPackageController):
         item.create(pkg)
 

ckan/logic/validators.py

@@ -265,18 +265,15 @@ def tag_not_uppercase(value, context):
     return value
 
 def tag_string_convert(key, data, errors, context):
+    '''Takes a list of tags that is a comma-separated string (in data[key])
+    and parses tag names. These are added to the data dict, enumerated. They
+    are also validated.'''
+    tag_string = data[key]
 
-    value = data[key]
-
-    # Ensure a tag string with only whitespace
-    # is converted to the empty list of tags.
-    # If we were to split(',') on this string,
-    # we'd get the non-empty list, [''].
-    if not value.strip():
-        return
+    tags = [tag.strip() \
+            for tag in tag_string.split(',') \
+            if tag.strip()]
 
-    tags = map(lambda s: s.strip(),
-               value.split(','))
     for num, tag in enumerate(tags):
         data[('tags', num, 'name')] = tag
 

ckan/plugins/interfaces.py

@@ -262,6 +262,16 @@ def after_search(self, search_results, search_params):
 
         return search_results
 
+    def before_index(self, pkg_dict):
+        '''
+             Extensions will recieve what will be given to the solr for indexing.
+             This is essentially a flattened dict (except for multlivlaued fields such as tags
+             of all the terms sent to the indexer.  The extension can modify this by returning
+             an altered version.
+        '''
+        return pkg_dict
+
+
 class IPluginObserver(Interface):
     """
     Plugin to the plugin loading mechanism

ckan/public/css/style.css

@@ -492,13 +492,13 @@ body.index.home .front-page .action-box-inner {
   min-height: 15em;
 }
 body.index.home .front-page .action-box-inner.collaborate {
-  background:url(/img/collaborate.png) no-repeat right top;
+  background:url(../img/collaborate.png) no-repeat right top;
 }
 body.index.home .front-page .action-box-inner.share {
-  background:url(/img/share.png) no-repeat right top;
+  background:url(../img/share.png) no-repeat right top;
 }
 body.index.home .front-page .action-box-inner.find {
-  background:url(/img/find.png) no-repeat right top;
+  background:url(../img/find.png) no-repeat right top;
 }
 body.index.home .front-page .action-box-inner a {
   font-weight: bold;
@@ -1170,7 +1170,7 @@ body.package.read h3 {
   display: block;
   margin-top: 4px;
   padding: 3px 22px 3px 10px;
-  background: url('/images/icons/arrow-right-16-black.png') no-repeat right;
+  background: url('../images/icons/arrow-right-16-black.png') no-repeat right;
   opacity:0.4;
   filter:alpha(opacity=40); /* For IE8 and earlier */
 }
@@ -1226,7 +1226,7 @@ body.package.read #sidebar li.widget-container {
 .notes {
   padding: 8px; 
   border-left: 2px solid #eee;
-  background: url('/images/ldquo.png') no-repeat top left #f7f7f7;
+  background: url('../images/ldquo.png') no-repeat top left #f7f7f7;
 }
 .notes #notes-toggle a {
   cursor: pointer;

ckan/public/scripts/application.js

@@ -508,7 +508,6 @@ CKAN.View.DatasetEditForm = Backbone.View.extend({
           var messageDiv = $('<div />').html(CKAN.Strings.youHaveUnsavedChanges).addClass('notice').hide();
           parentDiv.append(messageDiv);
           $('#unsaved-warning').append(parentDiv);
-          console.log($('#unsaved-warning'));
           messageDiv.show(200);
 
           boundToUnload = true;
@@ -859,13 +858,23 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
       el.attr('height', '100%');
       my.$dialog.append(el);
     }
+    // images
+    else if (resourceData.formatNormalized in {'png':'', 'jpg':'', 'gif':''} 
+        ||  resourceData.resource_type=='image') {
+      // we displays a fullscreen dialog with the url in an iframe.
+      my.$dialog.empty();
+      var el = $('<img />');
+      el.attr('src', resourceData.url);
+      el.css('max-width', '100%');
+      el.css('border', 'solid 4px black');
+      my.$dialog.append(el);
+    }
     else {
       // Cannot reliably preview this item - with no mimetype/format information, 
       // can't guarantee it's not a remote binary file such as an executable.
-      var _msg = $('<p class="error">We are unable to preview this type of resource: ' + resourceData.formatNormalized + '</p>');
       my.showError({
-        title: 'Unable to preview'
-        , message: _msg
+        title: 'Preview not available for data type: ' + resourceData.formatNormalized
+        , message: ''
       });
     }
   };
@@ -930,7 +939,10 @@ CKAN.View.ResourceAddLink = Backbone.View.extend({
   };
 
   my.showError = function (error) {
-    var _html = '<strong>' + $.trim(error.title) + '</strong><br />' + $.trim(error.message);
+    var _html = _.template(
+        '<div class="alert-message warning"><strong><%= title %></strong><br /><%= message %></div>'
+        , error
+        );
     my.$dialog.html(_html);
   };
 

ckan/public/scripts/vendor/recline/css/data-explorer.css

@@ -25,11 +25,6 @@
   width: 30px;
 }
 
-.doc-count {
-  font-weight: bold;
-  font-size: 120%;
-}
-
 .data-view-container {
   display: block;
   clear: both;
@@ -132,6 +127,11 @@
   * Data Table
   *********************************************************/
 
+.data-table-container {
+  overflow: auto;
+  height: 550px;
+}
+
 .data-table {
   border: 1px solid #ccc;
   font-size: 12px;

ckan/public/scripts/vendor/recline/recline.js

@@ -1045,7 +1045,7 @@ my.DataExplorer = Backbone.View.extend({
       </ul> \
       <div class="pagination"> \
         <form class="display-count"> \
-          Showing 0 to <input name="displayCount" type="text" value="{{displayCount}}" /> of  <span class="doc-count">{{docCount}}</span> \
+          Showing 0 to <input name="displayCount" type="text" value="{{displayCount}}" title="Edit and hit enter to change the number of rows displayed" /> of  <span class="doc-count">{{docCount}}</span> \
         </form> \
       </div> \
     </div> \
@@ -1067,7 +1067,7 @@ my.DataExplorer = Backbone.View.extend({
     var self = this;
     this.el = $(this.el);
     this.config = _.extend({
-        displayCount: 10
+        displayCount: 50
         , readOnly: false
       },
       options.config);