Merge branch 'master' of github.com:okfn/ckan into 847-new-theming-docs

ckan/config/deployment.ini_tmpl

@@ -133,6 +133,12 @@ ckan.feeds.author_link =
 #ofs.aws_access_key_id = ....
 #ofs.aws_secret_access_key = ....
 
+## Datapusher settings
+
+# Make sure you have set up the DataStore
+
+datapusher.formats = csv
+datapusher.url = http://datapusher.ckan.org/
 
 ## Activity Streams Settings
 

ckan/config/middleware.py

@@ -338,8 +338,8 @@ def __call__(self, environ, start_response):
             key = ''.join([
                 environ['HTTP_USER_AGENT'],
                 environ['REMOTE_ADDR'],
-                environ['HTTP_ACCEPT_LANGUAGE'],
-                environ['HTTP_ACCEPT_ENCODING'],
+                environ.get('HTTP_ACCEPT_LANGUAGE', ''),
+                environ.get('HTTP_ACCEPT_ENCODING', ''),
             ])
             key = hashlib.md5(key).hexdigest()
             # store key/data here

ckan/config/solr/schema-2.0.xml

@@ -129,6 +129,7 @@
     <field name="title_string" type="string" indexed="true" stored="false" />
 
     <field name="data_dict" type="string" indexed="false" stored="true" />
+    <field name="validated_data_dict" type="string" indexed="false" stored="true" />
 
     <field name="_version_" type="string" indexed="true" stored="true"/>
 

ckan/lib/dictization/model_dictize.py

@@ -300,10 +300,6 @@ def package_dictize(pkg, context):
     result_dict['metadata_created'] = pkg.metadata_created.isoformat() \
         if pkg.metadata_created else None
 
-    if context.get('for_view'):
-        for item in plugins.PluginImplementations( plugins.IPackageController):
-            result_dict = item.before_view(result_dict)
-
     return result_dict
 
 def _get_members(context, group, member_type):

ckan/lib/formatters.py

@@ -80,6 +80,22 @@ def localised_nice_date(datetime_, show_date=False, with_hours=False):
 
     :rtype: sting
     '''
+
+    def months_between(date1, date2):
+        if date1 > date2:
+            date1, date2 = date2, date1
+        m1 = date1.year * 12 + date1.month
+        m2 = date2.year * 12 + date2.month
+        months = m2 - m1
+        if date1.day > date2.day:
+            months -= 1
+        elif date1.day == date2.day:
+            seconds1 = date1.hour * 3600 + date1.minute + date1.second
+            seconds2 = date2.hour * 3600 + date2.minute + date2.second
+            if seconds1 > seconds2:
+                months -= 1
+        return months
+
     if not show_date:
         now = datetime.datetime.now()
         date_diff = now - datetime_
@@ -98,9 +114,16 @@ def localised_nice_date(datetime_, show_date=False, with_hours=False):
                 return ungettext('{hours} hour ago', '{hours} hours ago',
                                  seconds / 3600).format(hours=seconds / 3600)
         # more than one day
-        if days < 31:
+        months = months_between(datetime_, now)
+
+        if months < 1:
             return ungettext('{days} day ago', '{days} days ago',
                              days).format(days=days)
+        if months < 13:
+            return ungettext('{months} month ago', '{months} months ago',
+                             months).format(months=months)
+        return ungettext('over {years} year ago', 'over {years} years ago',
+                         months / 12).format(years=months / 12)
     # actual date
     details = {
         'min': datetime_.minute,

ckan/lib/helpers.py

@@ -48,6 +48,28 @@
 log = logging.getLogger(__name__)
 
 
+def _datestamp_to_datetime(datetime_):
+    ''' Converts a datestamp to a datetime.  If a datetime is provided it
+    just gets returned.
+
+    :param datetime_: the timestamp
+    :type datetime_: string or datetime
+
+    :rtype: datetime
+    '''
+    if isinstance(datetime_, basestring):
+        try:
+            datetime_ = date_str_to_datetime(datetime_)
+        except TypeError:
+            return None
+        except ValueError:
+            return None
+    # check we are now a datetime
+    if not isinstance(datetime_, datetime.datetime):
+        return None
+    return datetime_
+
+
 def redirect_to(*args, **kw):
     '''A routes.redirect_to wrapper to retain the i18n settings'''
     kw['__ckan_no_root'] = True
@@ -793,15 +815,8 @@ def render_datetime(datetime_, date_format=None, with_hours=False):
 
     :rtype: string
     '''
-    if isinstance(datetime_, basestring):
-        try:
-            datetime_ = date_str_to_datetime(datetime_)
-        except TypeError:
-            return ''
-        except ValueError:
-            return ''
-    # check we are now a datetime
-    if not isinstance(datetime_, datetime.datetime):
+    datetime_ = _datestamp_to_datetime(datetime_)
+    if not datetime_:
         return ''
     # if date_format was supplied we use it
     if date_format:
@@ -910,15 +925,33 @@ def dst(self, dt):
     def tzname(self, dt):
         return None
 
-
+@maintain.deprecated('h.time_ago_in_words_from_str is deprecated in 2.2 '
+                     'and will be removed.  Please use '
+                     'h.time_ago_from_timestamp instead')
 def time_ago_in_words_from_str(date_str, granularity='month'):
+    '''Deprecated in 2.2 use time_ago_from_timestamp'''
     if date_str:
         return date.time_ago_in_words(date_str_to_datetime(date_str),
                                       granularity=granularity)
     else:
         return _('Unknown')
 
 
+def time_ago_from_timestamp(timestamp):
+    ''' Returns a string like `5 months ago` for a datetime relative to now
+    :param timestamp: the timestamp or datetime
+    :type timestamp: string or datetime
+
+    :rtype: string
+    '''
+    datetime_ = _datestamp_to_datetime(timestamp)
+    if not datetime_:
+        return _('Unknown')
+
+    # the localised date
+    return formatters.localised_nice_date(datetime_, show_date=False)
+
+
 def button_attr(enable, type='primary'):
     if enable:
         return 'class="btn %s"' % type
@@ -1681,6 +1714,7 @@ def new_activities():
     'localised_filesize',
     'list_dict_filter',
     'new_activities',
+    'time_ago_from_timestamp',
     # imported into ckan.lib.helpers
     'literal',
     'link_to',

ckan/lib/navl/dictization_functions.py

@@ -1,6 +1,7 @@
 import copy
 import formencode as fe
 import inspect
+import json
 from pylons import config
 
 from ckan.common import _
@@ -402,3 +403,11 @@ def unflatten(data):
         unflattened[key] = [unflattened[key][s] for s in sorted(unflattened[key])]
 
     return unflattened
+
+
+class MissingNullEncoder(json.JSONEncoder):
+    '''json encoder that treats missing objects as null'''
+    def default(self, obj):
+        if isinstance(obj, Missing):
+            return None
+        return json.JSONEncoder.default(self, obj)

ckan/lib/search/__init__.py

@@ -123,7 +123,8 @@ def notify(self, entity, operation):
             dispatch_by_operation(
                 entity.__class__.__name__,
                 logic.get_action('package_show')(
-                    {'model': model, 'ignore_auth': True, 'validate': False},
+                    {'model': model, 'ignore_auth': True, 'validate': False,
+                     'use_cache': False},
                     {'id': entity.id}),
                 operation
             )
@@ -147,18 +148,18 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def
     log.info("Rebuilding search index...")
 
     package_index = index_for(model.Package)
+    context = {'model': model, 'ignore_auth': True, 'validate': False,
+        'use_cache': False}
 
     if package_id:
-        pkg_dict = logic.get_action('package_show')(
-            {'model': model, 'ignore_auth': True, 'validate': False},
+        pkg_dict = logic.get_action('package_show')(context,
             {'id': package_id})
         log.info('Indexing just package %r...', pkg_dict['name'])
         package_index.remove_dict(pkg_dict)
         package_index.insert_dict(pkg_dict)
     elif package_ids:
         for package_id in package_ids:
-            pkg_dict = logic.get_action('package_show')(
-                {'model': model, 'ignore_auth': True, 'validate': False},
+            pkg_dict = logic.get_action('package_show')(context,
                 {'id': package_id})
             log.info('Indexing just package %r...', pkg_dict['name'])
             package_index.update_dict(pkg_dict, True)
@@ -185,9 +186,7 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def
         for pkg_id in package_ids:
             try:
                 package_index.update_dict(
-                    logic.get_action('package_show')(
-                        {'model': model, 'ignore_auth': True,
-                         'validate': False},
+                    logic.get_action('package_show')(context,
                         {'id': pkg_id}
                     ),
                     defer_commit

ckan/lib/search/index.py

@@ -18,9 +18,13 @@
 from ckan.plugins import (PluginImplementations,
                           IPackageController)
 import ckan.logic as logic
+import ckan.lib.plugins as lib_plugins
+import ckan.lib.navl.dictization_functions
 
 log = logging.getLogger(__name__)
 
+_validate = ckan.lib.navl.dictization_functions.validate
+
 TYPE_FIELD = "entity_type"
 PACKAGE_TYPE = "package"
 KEY_CHARS = string.digits + string.letters + "_-"
@@ -102,8 +106,19 @@ def update_dict(self, pkg_dict, defer_commit=False):
     def index_package(self, pkg_dict, defer_commit=False):
         if pkg_dict is None:
             return
+
         pkg_dict['data_dict'] = json.dumps(pkg_dict)
 
+        if config.get('ckan.cache_validated_datasets', True):
+            package_plugin = lib_plugins.lookup_package_plugin(
+                pkg_dict.get('type'))
+
+            schema = package_plugin.show_package_schema()
+            validated_pkg_dict, errors = _validate(pkg_dict, schema, {
+                'model': model, 'session': model.Session})
+            pkg_dict['validated_data_dict'] = json.dumps(validated_pkg_dict,
+                cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)
+
         # add to string field for sorting
         title = pkg_dict.get('title')
         if title:

ckan/lib/search/query.py

@@ -279,11 +279,12 @@ def get_index(self,reference):
             data = json.loads(solr_response)
 
             if data['response']['numFound'] == 0:
-             raise SearchError('Dataset not found in the search index: %s' % reference)
+                raise SearchError('Dataset not found in the search index: %s' % reference)
             else:
                 return data['response']['docs'][0]
         except Exception, e:
-            log.exception(e)
+            if not isinstance(e, SearchError):
+                log.exception(e)
             raise SearchError(e)
         finally:
             conn.close()

ckan/logic/action/create.py

@@ -155,8 +155,9 @@ def package_create(context, data_dict):
     admins = []
     if user:
         user_obj = model.User.by_name(user.decode('utf8'))
-        admins = [user_obj]
-        data['creator_user_id'] = user_obj.id
+        if user_obj:
+            admins = [user_obj]
+            data['creator_user_id'] = user_obj.id
 
     pkg = model_save.package_dict_save(data, context)