Merge branch 'master' into clean-models

ckan/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '1.6.1b'
+__version__ = '1.8a'
 __description__ = 'Comprehensive Knowledge Archive Network (CKAN) Software'
 __long_description__ = \
 '''CKAN software provides a hub for datasets. The flagship site running CKAN 

ckan/config/solr/CHANGELOG.txt

@@ -1,6 +1,14 @@
 CKAN SOLR schemas changelog
 ===========================
 
+v1.4 - (ckan>=1.7)
+--------------------
+* Add Ascii folding filter to text fields.
+* Add capacity field for public, private access.
+* Add title_string so you can sort alphabetically on title.
+* Fields related to analytics, access and view counts.
+* Add data_dict field for the whole package_dict.
+
 v1.3 - (ckan>=1.5.1)
 --------------------
 * Use the index_id (hash of dataset id + site_id) as uniqueKey (#1430)

ckan/config/solr/schema-1.4.xml

@@ -51,6 +51,7 @@
             <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
             <filter class="solr.LowerCaseFilterFactory"/>
             <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+            <filter class="solr.ASCIIFoldingFilterFactory"/>
         </analyzer>
         <analyzer type="query">
             <tokenizer class="solr.WhitespaceTokenizerFactory"/>
@@ -63,6 +64,7 @@
             <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
             <filter class="solr.LowerCaseFilterFactory"/>
             <filter class="solr.SnowballPorterFilterFactory" language="English" protected="protwords.txt"/>
+            <filter class="solr.ASCIIFoldingFilterFactory"/>
         </analyzer>
     </fieldType>
 
@@ -115,6 +117,8 @@
     <field name="tags" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="groups" type="string" indexed="true" stored="true" multiValued="true"/>
 
+    <field name="capacity" type="string" indexed="true" stored="true" multiValued="false"/>
+
     <field name="res_description" type="textgen" indexed="true" stored="true" multiValued="true"/>
     <field name="res_format" type="string" indexed="true" stored="true" multiValued="true"/>
     <field name="res_url" type="string" indexed="true" stored="true" multiValued="true"/>
@@ -134,8 +138,8 @@
     <field name="parent_of" type="text" indexed="true" stored="false" multiValued="true"/>
     <field name="views_total" type="int" indexed="true" stored="false"/>
     <field name="views_recent" type="int" indexed="true" stored="false"/>
-    <field name="recources_accessed_total" type="int" indexed="true" stored="false"/>
-    <field name="recources_accessed_recent" type="int" indexed="true" stored="false"/>
+    <field name="resources_accessed_total" type="int" indexed="true" stored="false"/>
+    <field name="resources_accessed_recent" type="int" indexed="true" stored="false"/>
 
     <field name="metadata_created" type="date" indexed="true" stored="true" multiValued="false"/>
     <field name="metadata_modified" type="date" indexed="true" stored="true" multiValued="false"/>
@@ -144,8 +148,9 @@
 
     <!-- Copy the title field into titleString, and treat as a string
          (rather than text type).  This allows us to sort on the titleString -->
-    <field name="titleString" type="string" indexed="true" stored="false" />
-    <copyField source="title" dest="titleString"/>
+    <field name="title_string" type="string" indexed="true" stored="false" />
+
+    <field name="data_dict" type="string" indexed="false" stored="true" />
 
     <dynamicField name="extras_*" type="text" indexed="true" stored="true" multiValued="false"/>
     <dynamicField name="*" type="string" indexed="true"  stored="false"/>

ckan/controllers/package.py

@@ -480,6 +480,7 @@ def edit(self, id, data=None, errors=None, error_summary=None):
         c.errors_json = json.dumps(errors)
 
         self._setup_template_variables(context, {'id': id}, package_type=package_type)
+        c.related_count = len(c.pkg.related)
 
         # TODO: This check is to maintain backwards compatibility with the old way of creating
         # custom forms. This behaviour is now deprecated.
@@ -749,6 +750,8 @@ def resource_read(self, id, resource_id):
             c.package['isopen'] = False
         c.datastore_api = h.url_for('datastore_read', id=c.resource.get('id'),
                 qualified=True)
+
+        c.related_count = len(c.pkg.related)
         return render('package/resource_read.html')
 
     def resource_embedded_dataviewer(self, id, resource_id):

ckan/lib/alphabet_paginate.py

@@ -1,6 +1,7 @@
 '''
-Based on webhelpers.paginator, but each page is for items beginning
- with a particular letter.
+Based on webhelpers.paginator, but:
+ * each page is for items beginning with a particular letter
+ * output is suitable for Bootstrap
 
  Example:
         c.page = h.Page(
@@ -43,43 +44,55 @@ def __init__(self, collection, alpha_attribute, page, other_text, paging_thresho
         self.other_text = other_text
         self.paging_threshold = paging_threshold
         self.controller_name = controller_name
-        self.available = dict( (c,0,) for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ" )
+
+        self.letters = [char for char in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'] + [self.other_text]
+
+        # Work out which alphabet letters are 'available' i.e. have some results
+        # because we grey-out those which aren't.
+        self.available = dict( (c,0,) for c in self.letters )
         for c in self.collection:
             if isinstance(c, unicode):
                 x = c[0]
             elif isinstance(c, dict):
                 x = c[self.alpha_attribute][0]
             else:
                 x = getattr(c, self.alpha_attribute)[0]
+            x = x.upper()
+            if x not in self.letters:
+                x = self.other_text
             self.available[x] = self.available.get(x, 0) + 1
 
     def pager(self, q=None):
         '''Returns pager html - for navigating between the pages.
            e.g. Something like this:
-             <div class='pager'>
-                 <span class="pager_curpage">A</span>
-                 <a class="pager_link" href="/package/list?page=B">B</a>
-                 <a class="pager_link" href="/package/list?page=C">C</a>
+             <ul class='pagination pagination-alphabet'>
+                 <li class="active"><a href="/package/list?page=A">A</a></li>
+                 <li><a href="/package/list?page=B">B</a></li>
+                 <li><a href="/package/list?page=C">C</a></li>
                     ...
-                 <a class="pager_link" href="/package/list?page=Z">Z</a
-                 <a class="pager_link" href="/package/list?page=Other">Other</a
-             </div>
+                 <li class="disabled"><a href="/package/list?page=Z">Z</a></li>
+                 <li><a href="/package/list?page=Other">Other</a></li>
+             </ul>
         '''
         if self.item_count < self.paging_threshold:
             return ''
         pages = []
         page = q or self.page
-        letters = [char for char in 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'] + [self.other_text]
-        for letter in letters:
+        for letter in self.letters:
+            href = url_for(controller=self.controller_name, action='index', page=letter)
+            link = HTML.a(href=href, c=letter)
             if letter != page:
                 if self.available.get(letter, 0):
-                    page_element = HTML.a(class_='pager_link', href=url_for(controller=self.controller_name, action='index', page=letter),c=letter)
+                    li_class = ''
                 else:
-                    page_element = HTML.span(class_="pager_empty", c=letter)
+                    li_class = 'disabled'
             else:
-                page_element = HTML.span(class_='pager_curpage', c=letter)
+                li_class = 'active'
+            attributes = {'class_': li_class} if li_class else {}
+            page_element = HTML.li(link, **attributes)
             pages.append(page_element)
-        div = HTML.tag('div', class_='pager', *pages)
+        ul = HTML.tag('ul', *pages)
+        div = HTML.div(ul, class_='pagination pagination-alphabet')
         return div
 
 

ckan/lib/search/__init__.py

@@ -26,7 +26,7 @@ def text_traceback():
 
 SIMPLE_SEARCH = config.get('ckan.simple_search', False)
 
-SUPPORTED_SCHEMA_VERSIONS = ['1.3']
+SUPPORTED_SCHEMA_VERSIONS = ['1.4']
 
 DEFAULT_OPTIONS = {
     'limit': 20,

ckan/lib/search/index.py

@@ -99,6 +99,11 @@ def index_package(self, pkg_dict):
         if pkg_dict is None:
             return
 
+        # add to string field for sorting
+        title = pkg_dict.get('title')
+        if title:
+            pkg_dict['title_string'] = title
+
         if (not pkg_dict.get('state')) or ('active' not in pkg_dict.get('state')):
             return self.delete_package(pkg_dict)
 
@@ -163,7 +168,7 @@ def index_package(self, pkg_dict):
 
         pkg_dict = dict([(k.encode('ascii', 'ignore'), v) for (k, v) in pkg_dict.items()])
 
-        for k in ('title','notes'):
+        for k in ('title', 'notes', 'title_string'):
             if k in pkg_dict and pkg_dict[k]:
                 pkg_dict[k] = escape_xml_illegal_chars(pkg_dict[k])
 

ckan/logic/auth/delete.py

@@ -27,6 +27,15 @@ def related_delete(context, data_dict):
 
     related = get_related_object(context, data_dict)
     userobj = model.User.get( user )
+
+    if related.datasets:
+        package = related.datasets[0]
+
+        pkg_dict = { 'id': package.id }
+        authorized = package_delete(context, pkg_dict).get('success')
+        if authorized:
+            return {'success': True}
+
     if not userobj or userobj.id != related.owner_id:
         return {'success': False, 'msg': _('Only the owner can delete a related item')}
 

ckan/logic/auth/publisher/delete.py

@@ -40,6 +40,12 @@ def related_delete(context, data_dict):
 
     related = get_related_object(context, data_dict)
     userobj = model.User.get( user )
+
+    if related.datasets:
+        package = related.datasets[0]
+        if _groups_intersect( userobj.get_groups('organization'), package.get_groups('organization') ):
+            return {'success': True}
+
     if not userobj or userobj.id != related.owner_id:
         return {'success': False, 'msg': _('Only the owner can delete a related item')}
 

ckan/plugins/core.py

@@ -6,11 +6,10 @@
 from inspect import isclass
 from itertools import chain
 from pkg_resources import iter_entry_points
-from pyutilib.component.core import PluginGlobals, ExtensionPoint as PluginImplementations, implements
+from pyutilib.component.core import PluginGlobals, implements
+from pyutilib.component.core import ExtensionPoint as PluginImplementations
 from pyutilib.component.core import SingletonPlugin as _pca_SingletonPlugin
 from pyutilib.component.core import Plugin as _pca_Plugin
-from pyutilib.component.core import PluginEnvironment
-from sqlalchemy.orm.interfaces import MapperExtension
 
 from ckan.plugins.interfaces import IPluginObserver
 
@@ -23,18 +22,20 @@
 
 log = logging.getLogger(__name__)
 
-# Entry point group. 
+# Entry point group.
 PLUGINS_ENTRY_POINT_GROUP = "ckan.plugins"
 
 # Entry point group for system plugins (those that are part of core ckan and do
 # not need to be explicitly enabled by the user)
 SYSTEM_PLUGINS_ENTRY_POINT_GROUP = "ckan.system_plugins"
 
+
 class PluginNotFoundException(Exception):
     """
     Raised when a requested plugin cannot be found.
     """
 
+
 class Plugin(_pca_Plugin):
     """
     Base class for plugins which require multiple instances.
@@ -43,6 +44,7 @@ class Plugin(_pca_Plugin):
     probably use SingletonPlugin.
     """
 
+
 class SingletonPlugin(_pca_SingletonPlugin):
     """
     Base class for plugins which are singletons (ie most of them)
@@ -52,6 +54,7 @@ class SingletonPlugin(_pca_SingletonPlugin):
     same singleton instance.
     """
 
+
 def _get_service(plugin):
     """
     Return a service (ie an instance of a plugin class).
@@ -100,13 +103,15 @@ def load_all(config):
     for plugin in plugins:
         load(plugin)
 
+
 def reset():
     """
     Clear and reload all configured plugins
     """
     from pylons import config
     load_all(config)
 
+
 def load(plugin):
     """
     Load a single plugin, given a plugin name, class or instance
@@ -120,6 +125,7 @@ def load(plugin):
         observer_plugin.after_load(service)
     return service
 
+
 def unload_all():
     """
     Unload (deactivate) all loaded plugins
@@ -128,6 +134,7 @@ def unload_all():
         for service in env.services.copy():
             unload(service)
 
+
 def unload(plugin):
     """
     Unload a single plugin, given a plugin name, class or instance
@@ -144,6 +151,7 @@ def unload(plugin):
 
     return service
 
+
 def find_user_plugins(config):
     """
     Return all plugins specified by the user in the 'ckan.plugins' config
@@ -159,15 +167,15 @@ def find_user_plugins(config):
         plugins.extend(ep.load() for ep in entry_points)
     return plugins
 
+
 def find_system_plugins():
     """
     Return all plugins in the ckan.system_plugins entry point group.
-    
+
     These are essential for operation and therefore cannot be enabled/disabled
     through the configuration file.
     """
     return (
         ep.load()
         for ep in iter_entry_points(group=SYSTEM_PLUGINS_ENTRY_POINT_GROUP)
     )
-