Merge branch 'master' of github.com:okfn/ckan into 1199-fix-package-o…

…wner-org-edit

ckan/config/deployment.ini_tmpl

@@ -133,6 +133,21 @@ ckan.feeds.author_link =
 #ofs.aws_access_key_id = ....
 #ofs.aws_secret_access_key = ....
 
+# 'Bucket' to use for file storage
+#ckan.storage.bucket = default
+
+# Prefix for uploaded files (only used for pairtree)
+#ckan.storage.key_prefix = file/
+
+# The maximum content size, in bytes, for uploads
+#ckan.storage.max_content_length = 50000000
+
+## Datapusher settings
+
+# Make sure you have set up the DataStore
+
+datapusher.formats = csv
+datapusher.url = http://datapusher.ckan.org/
 
 ## Activity Streams Settings
 

ckan/config/solr/schema-2.0.xml

@@ -129,6 +129,7 @@
     <field name="title_string" type="string" indexed="true" stored="false" />
 
     <field name="data_dict" type="string" indexed="false" stored="true" />
+    <field name="validated_data_dict" type="string" indexed="false" stored="true" />
 
     <field name="_version_" type="string" indexed="true" stored="true"/>
 

ckan/lib/dictization/model_dictize.py

@@ -300,10 +300,6 @@ def package_dictize(pkg, context):
     result_dict['metadata_created'] = pkg.metadata_created.isoformat() \
         if pkg.metadata_created else None
 
-    if context.get('for_view'):
-        for item in plugins.PluginImplementations( plugins.IPackageController):
-            result_dict = item.before_view(result_dict)
-
     return result_dict
 
 def _get_members(context, group, member_type):

ckan/lib/navl/dictization_functions.py

@@ -1,6 +1,7 @@
 import copy
 import formencode as fe
 import inspect
+import json
 from pylons import config
 
 from ckan.common import _
@@ -402,3 +403,11 @@ def unflatten(data):
         unflattened[key] = [unflattened[key][s] for s in sorted(unflattened[key])]
 
     return unflattened
+
+
+class MissingNullEncoder(json.JSONEncoder):
+    '''json encoder that treats missing objects as null'''
+    def default(self, obj):
+        if isinstance(obj, Missing):
+            return None
+        return json.JSONEncoder.default(self, obj)

ckan/lib/search/__init__.py

@@ -123,7 +123,8 @@ def notify(self, entity, operation):
             dispatch_by_operation(
                 entity.__class__.__name__,
                 logic.get_action('package_show')(
-                    {'model': model, 'ignore_auth': True, 'validate': False},
+                    {'model': model, 'ignore_auth': True, 'validate': False,
+                     'use_cache': False},
                     {'id': entity.id}),
                 operation
             )
@@ -147,18 +148,18 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def
     log.info("Rebuilding search index...")
 
     package_index = index_for(model.Package)
+    context = {'model': model, 'ignore_auth': True, 'validate': False,
+        'use_cache': False}
 
     if package_id:
-        pkg_dict = logic.get_action('package_show')(
-            {'model': model, 'ignore_auth': True, 'validate': False},
+        pkg_dict = logic.get_action('package_show')(context,
             {'id': package_id})
         log.info('Indexing just package %r...', pkg_dict['name'])
         package_index.remove_dict(pkg_dict)
         package_index.insert_dict(pkg_dict)
     elif package_ids:
         for package_id in package_ids:
-            pkg_dict = logic.get_action('package_show')(
-                {'model': model, 'ignore_auth': True, 'validate': False},
+            pkg_dict = logic.get_action('package_show')(context,
                 {'id': package_id})
             log.info('Indexing just package %r...', pkg_dict['name'])
             package_index.update_dict(pkg_dict, True)
@@ -185,9 +186,7 @@ def rebuild(package_id=None, only_missing=False, force=False, refresh=False, def
         for pkg_id in package_ids:
             try:
                 package_index.update_dict(
-                    logic.get_action('package_show')(
-                        {'model': model, 'ignore_auth': True,
-                         'validate': False},
+                    logic.get_action('package_show')(context,
                         {'id': pkg_id}
                     ),
                     defer_commit

ckan/lib/search/index.py

@@ -18,9 +18,13 @@
 from ckan.plugins import (PluginImplementations,
                           IPackageController)
 import ckan.logic as logic
+import ckan.lib.plugins as lib_plugins
+import ckan.lib.navl.dictization_functions
 
 log = logging.getLogger(__name__)
 
+_validate = ckan.lib.navl.dictization_functions.validate
+
 TYPE_FIELD = "entity_type"
 PACKAGE_TYPE = "package"
 KEY_CHARS = string.digits + string.letters + "_-"
@@ -102,8 +106,19 @@ def update_dict(self, pkg_dict, defer_commit=False):
     def index_package(self, pkg_dict, defer_commit=False):
         if pkg_dict is None:
             return
+
         pkg_dict['data_dict'] = json.dumps(pkg_dict)
 
+        if config.get('ckan.cache_validated_datasets', True):
+            package_plugin = lib_plugins.lookup_package_plugin(
+                pkg_dict.get('type'))
+
+            schema = package_plugin.show_package_schema()
+            validated_pkg_dict, errors = _validate(pkg_dict, schema, {
+                'model': model, 'session': model.Session})
+            pkg_dict['validated_data_dict'] = json.dumps(validated_pkg_dict,
+                cls=ckan.lib.navl.dictization_functions.MissingNullEncoder)
+
         # add to string field for sorting
         title = pkg_dict.get('title')
         if title:

ckan/lib/search/query.py

@@ -279,11 +279,12 @@ def get_index(self,reference):
             data = json.loads(solr_response)
 
             if data['response']['numFound'] == 0:
-             raise SearchError('Dataset not found in the search index: %s' % reference)
+                raise SearchError('Dataset not found in the search index: %s' % reference)
             else:
                 return data['response']['docs'][0]
         except Exception, e:
-            log.exception(e)
+            if not isinstance(e, SearchError):
+                log.exception(e)
             raise SearchError(e)
         finally:
             conn.close()

ckan/logic/action/create.py

@@ -155,8 +155,9 @@ def package_create(context, data_dict):
     admins = []
     if user:
         user_obj = model.User.by_name(user.decode('utf8'))
-        admins = [user_obj]
-        data['creator_user_id'] = user_obj.id
+        if user_obj:
+            admins = [user_obj]
+            data['creator_user_id'] = user_obj.id
 
     pkg = model_save.package_dict_save(data, context)
 

ckan/logic/action/get.py

@@ -4,6 +4,7 @@
 import logging
 import json
 import datetime
+import socket
 
 from pylons import config
 import sqlalchemy
@@ -768,7 +769,37 @@ def package_show(context, data_dict):
 
     _check_access('package_show', context, data_dict)
 
-    package_dict = model_dictize.package_dictize(pkg, context)
+    package_dict = None
+    use_cache = (context.get('use_cache', True)
+        and not 'revision_id' in context
+        and not 'revision_date' in context)
+    if use_cache:
+        try:
+            search_result = search.show(name_or_id)
+        except (search.SearchError, socket.error):
+            pass
+        else:
+            use_validated_cache = 'schema' not in context
+            if use_validated_cache and 'validated_data_dict' in search_result:
+                package_dict = json.loads(search_result['validated_data_dict'])
+                package_dict_validated = True
+            else:
+                package_dict = json.loads(search_result['data_dict'])
+                package_dict_validated = False
+            metadata_modified = pkg.metadata_modified.isoformat()
+            search_metadata_modified = search_result['metadata_modified']
+            # solr stores less precice datetime,
+            # truncate to 22 charactors to get good enough match
+            if metadata_modified[:22] != search_metadata_modified[:22]:
+                package_dict = None
+
+    if not package_dict:
+        package_dict = model_dictize.package_dictize(pkg, context)
+        package_dict_validated = False
+
+    if context.get('for_view'):
+        for item in plugins.PluginImplementations(plugins.IPackageController):
+            package_dict = item.before_view(package_dict)
 
     for item in plugins.PluginImplementations(plugins.IPackageController):
         item.read(pkg)
@@ -777,14 +808,15 @@ def package_show(context, data_dict):
         for item in plugins.PluginImplementations(plugins.IResourceController):
             resource_dict = item.before_show(resource_dict)
 
-    package_plugin = lib_plugins.lookup_package_plugin(package_dict['type'])
-    if 'schema' in context:
-        schema = context['schema']
-    else:
-        schema = package_plugin.show_package_schema()
-
-    if schema and context.get('validate', True):
-        package_dict, errors = _validate(package_dict, schema, context=context)
+    if not package_dict_validated:
+        package_plugin = lib_plugins.lookup_package_plugin(package_dict['type'])
+        if 'schema' in context:
+            schema = context['schema']
+        else:
+            schema = package_plugin.show_package_schema()
+            if schema and context.get('validate', True):
+                package_dict, errors = _validate(package_dict, schema,
+                    context=context)
 
     for item in plugins.PluginImplementations(plugins.IPackageController):
         item.after_show(context, package_dict)
@@ -880,6 +912,10 @@ def _group_or_org_show(context, data_dict, is_org=False):
 
     if group is None:
         raise NotFound
+    if is_org and not group.is_organization:
+        raise NotFound
+    if not is_org and group.is_organization:
+        raise NotFound
 
     if is_org:
         _check_access('organization_show',context, data_dict)
@@ -2149,8 +2185,7 @@ def activity_detail_list(context, data_dict):
     # authorized to read.
     model = context['model']
     activity_id = _get_or_bust(data_dict, 'id')
-    activity_detail_objects = model.Session.query(
-        model.activity.ActivityDetail).filter_by(activity_id=activity_id).all()
+    activity_detail_objects = model.ActivityDetail.by_activity_id(activity_id)
     return model_dictize.activity_detail_list_dictize(activity_detail_objects, context)
 
 

ckan/logic/auth/create.py

@@ -4,10 +4,10 @@
 from ckan.common import _
 
 
-@logic.auth_sysadmins_check
 def package_create(context, data_dict=None):
     user = context['user']
-    if not new_authz.auth_is_registered_user():
+
+    if new_authz.auth_is_anon_user(context):
         check1 = new_authz.check_config_permission('anon_create_dataset')
     else:
         check1 = new_authz.check_config_permission('create_dataset_if_not_in_organization') \
@@ -32,7 +32,7 @@ def package_create(context, data_dict=None):
 
 def file_upload(context, data_dict=None):
     user = context['user']
-    if not new_authz.auth_is_registered_user():
+    if new_authz.auth_is_anon_user(context):
         return {'success': False, 'msg': _('User %s not authorized to create packages') % user}
     return {'success': True}
 

ckan/logic/auth/update.py

@@ -23,7 +23,7 @@ def package_update(context, data_dict):
         )
     else:
         # If dataset is not owned then we can edit if config permissions allow
-        if new_authz.auth_is_registered_user():
+        if not new_authz.auth_is_anon_user(context):
             check1 = new_authz.check_config_permission(
                 'create_dataset_if_not_in_organization')
         else:

ckan/model/activity.py

@@ -2,6 +2,7 @@
 
 from sqlalchemy import orm, types, Column, Table, ForeignKey, desc, or_
 
+import ckan.model
 import meta
 import types as _types
 import domain_object
@@ -62,6 +63,11 @@ def __init__(self, activity_id, object_id, object_type, activity_type,
         else:
             self.data = data
 
+    @classmethod
+    def by_activity_id(cls, activity_id):
+        return ckan.model.Session.query(cls) \
+                .filter_by(activity_id = activity_id).all()
+
 
 meta.mapper(ActivityDetail, activity_detail_table, properties = {
     'activity':orm.relation ( Activity, backref=orm.backref('activity_detail'))

ckan/new_authz.py

@@ -9,6 +9,8 @@
 import ckan.model as model
 from ckan.common import OrderedDict, _, c
 
+import ckan.lib.maintain as maintain
+
 log = getLogger(__name__)
 
 
@@ -334,11 +336,34 @@ def check_config_permission(permission):
         return CONFIG_PERMISSIONS[permission]
     return False
 
-
+@maintain.deprecated('Use auth_is_loggedin_user instead')
 def auth_is_registered_user():
+    '''
+    This function is deprecated, please use the auth_is_loggedin_user instead
+    '''
+    return auth_is_loggedin_user()
+
+def auth_is_loggedin_user():
     ''' Do we have a logged in user '''
     try:
         context_user = c.user
     except TypeError:
         context_user = None
     return bool(context_user)
+
+def auth_is_anon_user(context):
+    ''' Is this an anonymous user?
+        eg Not logged in if a web request and not user defined in context
+        if logic functions called directly
+
+        See ckan/lib/base.py:232 for pylons context object logic
+    '''
+    try:
+        is_anon_user = (not bool(c.user) and bool(c.author))
+    except TypeError:
+        # No c object set, this is not a call done via the web interface,
+        # but directly, eg from an extension
+        context_user = context.get('user')
+        is_anon_user = not bool(context_user)
+
+    return is_anon_user