Merge branch 'release-v2.0' into idatasetform-fixes-branched-off-rele…

…ase-v2.0

ckan/controllers/tag.py

@@ -1,5 +1,5 @@
 from pylons.i18n import _
-from pylons import request, c
+from pylons import request, c, config
 
 import ckan.logic as logic
 import ckan.model as model
@@ -65,4 +65,7 @@ def read(self, id):
         except logic.NotFound:
             base.abort(404, _('Tag not found'))
 
-        return base.render('tag/read.html')
+        if h.asbool(config.get('ckan.legacy_templates', False)):
+            return base.render('tag/read.html')
+        else:
+            h.redirect_to(controller='package', action='search', tags=c.tag.get('name'))

ckan/lib/activity_streams_session_extension.py

@@ -1,7 +1,11 @@
+from pylons import config
 from sqlalchemy.orm.session import SessionExtension
+from paste.deploy.converters import asbool
 import logging
+
 logger = logging.getLogger(__name__)
 
+
 def activity_stream_item(obj, activity_type, revision, user_id):
     method = getattr(obj, "activity_stream_item", None)
     if callable(method):
@@ -11,6 +15,7 @@ def activity_stream_item(obj, activity_type, revision, user_id):
             "activity_stream_item() method, it must not be a package.")
         return None
 
+
 def activity_stream_detail(obj, activity_id, activity_type):
     method = getattr(obj, "activity_stream_detail",
             None)
@@ -21,6 +26,7 @@ def activity_stream_detail(obj, activity_id, activity_type):
             "activity_stream_detail() method.")
         return None
 
+
 class DatasetActivitySessionExtension(SessionExtension):
     """Session extension that emits activity stream activities for packages
     and related objects.
@@ -36,6 +42,8 @@ class DatasetActivitySessionExtension(SessionExtension):
 
     """
     def before_commit(self, session):
+        if not asbool(config.get('ckan.activity_streams_enabled', 'true')):
+            return
 
         session.flush()
 

ckan/lib/app_globals.py

@@ -109,9 +109,15 @@ def get_config_value(key, default=''):
             value = model.get_system_info(key)
         else:
             value = None
+        config_value = config.get(key)
+        # sort encodeings if needed
+        if isinstance(config_value, str):
+            try:
+                config_value = config_value.decode('utf-8')
+            except UnicodeDecodeError:
+                config_value = config_value.decode('latin-1')
         # we want to store the config the first time we get here so we can
         # reset them if needed
-        config_value = config.get(key)
         if key not in _CONFIG_CACHE:
             _CONFIG_CACHE[key] = config_value
         if value is not None:

ckan/lib/dictization/model_dictize.py

@@ -399,15 +399,25 @@ def tag_dictize(tag, context):
     tag_dict = d.table_dictize(tag, context)
     query = search.PackageSearchQuery()
 
-    q = {'q': '+tags:"%s" +capacity:public' % tag.name, 'fl': 'data_dict',
-         'wt': 'json', 'rows': 1000}
+    tag_query = u'+capacity:public '
+    vocab_id = tag_dict.get('vocabulary_id')
 
-    package_dicts = [h.json.loads(result['data_dict']) for result in query.run(q)['results']]
+    if vocab_id:
+        model = context['model']
+        vocab = model.Vocabulary.get(vocab_id)
+        tag_query += u'+vocab_{0}:"{1}"'.format(vocab.name, tag.name)
+    else:
+        tag_query += u'+tags:"{0}"'.format(tag.name)
+
+    q = {'q': tag_query, 'fl': 'data_dict', 'wt': 'json', 'rows': 1000}
+
+    package_dicts = [h.json.loads(result['data_dict'])
+                     for result in query.run(q)['results']]
 
     # Add display_names to tags. At first a tag's display_name is just the
     # same as its name, but the display_name might get changed later (e.g.
     # translated into another language by the multilingual extension).
-    assert not tag_dict.has_key('display_name')
+    assert 'display_name' not in tag_dict
     tag_dict['display_name'] = tag_dict['name']
 
     if context.get('for_view'):

ckan/lib/helpers.py

@@ -1129,13 +1129,16 @@ def add_url_param(alternative_url=None, controller=None, action=None,
     return _create_url_with_params(params=params, controller=controller,
                                    action=action, extras=extras)
 
-
 def remove_url_param(key, value=None, replace=None, controller=None,
                      action=None, extras=None, alternative_url=None):
-    ''' Remove a key from the current parameters. A specific key/value
-    pair can be removed by passing a second value argument otherwise all
-    pairs matching the key will be removed. If replace is given then a
-    new param key=replace will be added.
+    ''' Remove one or multiple keys from the current parameters.
+    The first parameter can be either a string with the name of the key to
+    remove or a list of keys to remove.
+    A specific key/value pair can be removed by passing a second value
+    argument otherwise all pairs matching the key will be removed. If replace
+    is given then a new param key=replace will be added.
+    Note that the value and replace parameters only apply to the first key
+    provided (or the only one provided if key is a string).
 
     controller action & extras (dict) are used to create the base url
     via url_for(controller=controller, action=action, **extras)
@@ -1144,14 +1147,20 @@ def remove_url_param(key, value=None, replace=None, controller=None,
     This can be overriden providing an alternative_url, which will be used
     instead.
     '''
+    if isinstance(key, basestring):
+      keys = [key]
+    else:
+      keys = key
+
     params_nopage = [(k, v) for k, v in request.params.items() if k != 'page']
     params = list(params_nopage)
     if value:
-        params.remove((key, value))
+        params.remove((keys[0], value))
     else:
-        [params.remove((k, v)) for (k, v) in params[:] if k == key]
+        for key in keys:
+          [params.remove((k, v)) for (k, v) in params[:] if k == key]
     if replace is not None:
-        params.append((key, replace))
+        params.append((keys[0], replace))
 
     if alternative_url:
         return _url_with_params(alternative_url, params)
@@ -1441,6 +1450,26 @@ def resource_preview(resource, pkg_id):
                    resource_url=url,
                    raw_resource_url=resource.get('url'))
 
+def list_dict_filter(list_, search_field, output_field, value):
+    ''' Takes a list of dicts and returns the value of a given key if the
+    item has a matching value for a supplied key
+
+    :param list_: the list to search through for matching items
+    :type list_: list of dicts
+
+    :param search_field: the key to use to find matching items
+    :type search_field: string
+
+    :param output_field: the key to use to output the value
+    :type output_field: string
+
+    :param value: the value to search for
+    '''
+
+    for item in list_:
+        if item.get(search_field) == value:
+            return item.get(output_field, value)
+    return value
 
 def SI_number_span(number):
     ''' outputs a span with the number in SI unit eg 14700 -> 14.7k '''
@@ -1539,6 +1568,7 @@ def SI_number_span(number):
            'localised_SI_number',
            'localised_nice_date',
            'localised_filesize',
+           'list_dict_filter',
            # imported into ckan.lib.helpers
            'literal',
            'link_to',

ckan/lib/search/index.py

@@ -157,12 +157,11 @@ def index_package(self, pkg_dict, defer_commit=False):
 
         # if there is an owner_org we want to add this to groups for index
         # purposes
-        if pkg_dict['owner_org'] and pkg_dict.get('organization'):
+        if pkg_dict.get('organization'):
            pkg_dict['organization'] = pkg_dict['organization']['name']
         else:
            pkg_dict['organization'] = None
 
-
         # tracking
         tracking_summary = pkg_dict.pop('tracking_summary', None)
         if tracking_summary:

ckan/logic/action/create.py

@@ -1,7 +1,9 @@
 import logging
+from pylons import config
 from pylons.i18n import _
 
 import ckan.new_authz as new_authz
+import ckan.lib.helpers as h
 import ckan.lib.plugins as lib_plugins
 import ckan.logic as logic
 import ckan.rating as ratings
@@ -422,13 +424,14 @@ def member_create(context, data_dict=None):
             filter(model.Member.table_name == obj_type).\
             filter(model.Member.table_id == obj_id).\
             filter(model.Member.group_id == group.id).\
-            filter(model.Member.state    == "active").first()
+            filter(model.Member.state == "active").first()
     if member:
         member.capacity = capacity
     else:
         member = model.Member(table_name = obj_type,
                               table_id = obj_id,
                               group_id = group.id,
+                              state = 'active',
                               capacity=capacity)
 
     model.Session.add(member)
@@ -888,6 +891,9 @@ def activity_create(context, activity_dict, ignore_auth=False):
     :rtype: dictionary
 
     '''
+    if not h.asbool(config.get('ckan.activity_streams_enabled', 'true')):
+        return
+
     model = context['model']
 
     # Any revision_id that the caller attempts to pass in the activity_dict is

ckan/logic/action/get.py

@@ -262,40 +262,33 @@ def member_list(context, data_dict=None):
     model = context['model']
 
     group = model.Group.get(_get_or_bust(data_dict, 'id'))
+    if not group:
+        raise NotFound
+
     obj_type = data_dict.get('object_type', None)
     capacity = data_dict.get('capacity', None)
 
     # User must be able to update the group to remove a member from it
     _check_access('group_show', context, data_dict)
 
     q = model.Session.query(model.Member).\
-            filter(model.Member.group_id == group.id).\
-            filter(model.Member.state    == "active")
+        filter(model.Member.group_id == group.id).\
+        filter(model.Member.state == "active")
 
     if obj_type:
         q = q.filter(model.Member.table_name == obj_type)
     if capacity:
         q = q.filter(model.Member.capacity == capacity)
 
-    lookup = {}
-    def type_lookup(name):
-        if name in lookup:
-            return lookup[name]
-        if hasattr(model, name.title()):
-            lookup[name] = getattr(model,name.title())
-            return lookup[name]
-        return None
-
     trans = new_authz.roles_trans()
+
     def translated_capacity(capacity):
         try:
             return trans[capacity]
         except KeyError:
             return capacity
 
-    return [(m.table_id,
-             type_lookup(m.table_name),
-             translated_capacity(m.capacity),)
+    return [(m.table_id, m.table_name, translated_capacity(m.capacity))
             for m in q.all()]
 
 def _group_or_org_list(context, data_dict, is_org=False):

ckan/logic/validators.py

@@ -18,7 +18,7 @@ def owner_org_validator(key, data, errors, context):
 
     value = data.get(key)
 
-    if value is missing or value is None:
+    if value is missing or not value:
         if not ckan.new_authz.check_config_permission('create_unowned_dataset'):
             raise Invalid(_('A organization must be supplied'))
         data.pop(key, None)

ckan/migration/versions/067_turn_extras_to_strings.py

@@ -7,7 +7,7 @@ def upgrade(migrate_engine):
         revision_tables = 'package_extra_revision group_extra_revision'
 
         for table in tables.split():
-            sql = """select id, value from {table} where substr(value,0,1) = '"' """.format(table=table)
+            sql = """select id, value from {table} where substr(value,1,1) = '"' """.format(table=table)
             results = connection.execute(sql)
             for result in results:
                 id, value = result
@@ -16,7 +16,7 @@ def upgrade(migrate_engine):
                                    json.loads(value), id)
 
         for table in revision_tables.split():
-            sql = """select id, revision_id, value from {table} where substr(value,0,1) = '"' """.format(table=table)
+            sql = """select id, revision_id, value from {table} where substr(value,1,1) = '"' """.format(table=table)
 
             results = connection.execute(sql)
             for result in results:

ckan/misc.py

@@ -35,8 +35,8 @@ class MarkdownFormat(TextFormat):
     normal_link = re.compile('<(http:[^>]+)>')
 
     html_whitelist = 'b center li ol p table td tr ul'.split(' ')
-    whitelist_elem = re.compile(r'<(\/?(%s)[^>]*)>' % "|".join(html_whitelist), re.IGNORECASE)
-    whitelist_escp = re.compile(r'\\xfc\\xfd(\/?(%s)[^>]*?)\\xfd\\xfc' % "|".join(html_whitelist), re.IGNORECASE)
+    whitelist_elem = re.compile(r'<(\/?((%s)(\s[^>]*)?))>' % "|".join(html_whitelist), re.IGNORECASE)
+    whitelist_escp = re.compile(r'\\xfc\\xfd(\/?((%s)(\s[^>]*?)?))\\xfd\\xfc' % "|".join(html_whitelist), re.IGNORECASE)
     normal_link = re.compile(r'<a[^>]*?href="([^"]*?)"[^>]*?>', re.IGNORECASE)
     abbrev_link = re.compile(r'<(http://[^>]*)>', re.IGNORECASE)
     any_link = re.compile(r'<a[^>]*?>', re.IGNORECASE)

ckan/new_authz.py

@@ -49,6 +49,7 @@ def get_group_or_org_admin_ids(group_id):
     q = model.Session.query(model.Member) \
         .filter(model.Member.group_id == group_id) \
         .filter(model.Member.table_name == 'user') \
+        .filter(model.Member.state == 'active') \
         .filter(model.Member.capacity == 'admin')
     return [a.table_id for a in q.all()]
 
@@ -135,6 +136,7 @@ def has_user_permission_for_group_or_org(group_id, user_name, permission):
     q = model.Session.query(model.Member) \
         .filter(model.Member.group_id == group_id) \
         .filter(model.Member.table_name == 'user') \
+        .filter(model.Member.state == 'active') \
         .filter(model.Member.table_id == user_id)
     # see if any role has the required permission
     # admin permission allows anything for the group
@@ -158,6 +160,7 @@ def users_role_for_group_or_org(group_id, user_name):
     q = model.Session.query(model.Member) \
         .filter(model.Member.group_id == group_id) \
         .filter(model.Member.table_name == 'user') \
+        .filter(model.Member.state == 'active') \
         .filter(model.Member.table_id == user_id)
     # return the first role we find
     for row in q.all():
@@ -176,6 +179,7 @@ def has_user_permission_for_some_org(user_name, permission):
     # get any groups the user has with the needed role
     q = model.Session.query(model.Member) \
         .filter(model.Member.table_name == 'user') \
+        .filter(model.Member.state == 'active') \
         .filter(model.Member.capacity.in_(roles)) \
         .filter(model.Member.table_id == user_id)
     group_ids = []