Merge remote-tracking branch 'refs/remotes/ckan/master' into docker-u…

…pdate

ckan/controllers/group.py

@@ -294,13 +294,6 @@ def pager_url(q=None, page=None):
                     else:
                         search_extras[param] = value
 
-            include_private = False
-            user_member_of_orgs = [org['id'] for org
-                                   in h.organizations_available('read')]
-
-            if (c.group and c.group.id in user_member_of_orgs):
-                include_private = True
-
             facets = OrderedDict()
 
             default_facet_titles = {'organization': _('Organizations'),
@@ -318,16 +311,12 @@ def pager_url(q=None, page=None):
             # Facet titles
             self._update_facet_titles(facets, group_type)
 
-            if 'capacity' in facets and (group_type != 'organization' or
-                                         not user_member_of_orgs):
-                del facets['capacity']
-
             c.facet_titles = facets
 
             data_dict = {
                 'q': q,
                 'fq': '',
-                'include_private': include_private,
+                'include_private': True,
                 'facet.field': facets.keys(),
                 'rows': limit,
                 'sort': sort_by,

ckan/lib/cli.py

@@ -13,6 +13,7 @@
 import logging
 import urlparse
 from optparse import OptionConflictError
+import traceback
 
 import sqlalchemy as sa
 import routes
@@ -851,7 +852,7 @@ def add(self):
             user_dict = logic.get_action('user_create')(context, data_dict)
             pprint(user_dict)
         except logic.ValidationError, e:
-            error(e)
+            error(traceback.format_exc())
 
     def remove(self):
         import ckan.model as model
@@ -1491,10 +1492,8 @@ def profile_url(url):
                 print 'App error: ', url.strip()
             except KeyboardInterrupt:
                 raise
-            except:
-                import traceback
-                traceback.print_exc()
-                print 'Unknown error: ', url.strip()
+            except Exception:
+                error(traceback.format_exc())
 
         output_filename = 'ckan%s.profile' % re.sub('[/?]', '.', url.replace('/', '.'))
         profile_command = "profile_url('%s')" % url
@@ -2478,7 +2477,7 @@ def command(self):
                     config_filepath, options, self.options.section,
                     edit=self.options.edit)
             except config_tool.ConfigToolError, e:
-                error(e)
+                error(traceback.format_exc())
 
 
 class JobsCommand(CkanCommand):

ckan/lib/helpers.py

@@ -24,7 +24,7 @@
 import webhelpers.text as whtext
 import webhelpers.date as date
 from markdown import markdown
-from bleach import clean as clean_html
+from bleach import clean as clean_html, ALLOWED_TAGS, ALLOWED_ATTRIBUTES
 from pylons import url as _pylons_default_url
 from ckan.common import config, is_flask_request
 from flask import redirect as _flask_redirect
@@ -48,6 +48,15 @@
 
 log = logging.getLogger(__name__)
 
+MARKDOWN_TAGS = set([
+    'del', 'dd', 'dl', 'dt', 'h1', 'h2',
+    'h3', 'img', 'kbd', 'p', 'pre', 's',
+    'sup', 'sub', 'strike', 'br', 'hr'
+]).union(ALLOWED_TAGS)
+
+MARKDOWN_ATTRIBUTES = copy.deepcopy(ALLOWED_ATTRIBUTES)
+MARKDOWN_ATTRIBUTES.setdefault('img', []).extend(['src', 'alt', 'title'])
+
 
 class HelperAttributeDict(dict):
     def __init__(self, *args, **kwargs):
@@ -1859,7 +1868,9 @@ def render_markdown(data, auto_link=True, allow_html=False):
         data = markdown(data.strip())
     else:
         data = RE_MD_HTML_TAGS.sub('', data.strip())
-        data = markdown(clean_html(data, strip=True))
+        data = clean_html(
+            markdown(data), strip=True,
+            tags=MARKDOWN_TAGS, attributes=MARKDOWN_ATTRIBUTES)
     # tags can be added by tag:... or tag:"...." and a link will be made
     # from it
     if auto_link:

ckan/logic/action/get.py

@@ -818,6 +818,9 @@ def user_list(context, data_dict):
     :param order_by: which field to sort the list by (optional, default:
       ``'name'``). Can be any user field or ``edits`` (i.e. number_of_edits).
     :type order_by: string
+    :param all_fields: return full user dictionaries instead of just names.
+      (optional, default: ``True``)
+    :type all_fields: boolean
 
     :rtype: list of user dictionaries. User properties include:
       ``number_of_edits`` which counts the revisions by the user and
@@ -831,26 +834,30 @@ def user_list(context, data_dict):
 
     q = data_dict.get('q', '')
     order_by = data_dict.get('order_by', 'name')
+    all_fields = asbool(data_dict.get('all_fields', True))
 
-    query = model.Session.query(
-        model.User,
-        model.User.name.label('name'),
-        model.User.fullname.label('fullname'),
-        model.User.about.label('about'),
-        model.User.about.label('email'),
-        model.User.created.label('created'),
-        _select([_func.count(model.Revision.id)],
-                _or_(
-                    model.Revision.author == model.User.name,
-                    model.Revision.author == model.User.openid
-                )).label('number_of_edits'),
-        _select([_func.count(model.Package.id)],
-                _and_(
-                    model.Package.creator_user_id == model.User.id,
-                    model.Package.state == 'active',
-                    model.Package.private == False,
-                )).label('number_created_packages')
-    )
+    if all_fields:
+        query = model.Session.query(
+            model.User,
+            model.User.name.label('name'),
+            model.User.fullname.label('fullname'),
+            model.User.about.label('about'),
+            model.User.about.label('email'),
+            model.User.created.label('created'),
+            _select([_func.count(model.Revision.id)],
+                    _or_(
+                        model.Revision.author == model.User.name,
+                        model.Revision.author == model.User.openid
+                    )).label('number_of_edits'),
+            _select([_func.count(model.Package.id)],
+                    _and_(
+                        model.Package.creator_user_id == model.User.id,
+                        model.Package.state == 'active',
+                        model.Package.private == False,
+                    )).label('number_created_packages')
+        )
+    else:
+        query = model.Session.query(model.User.name)
 
     if q:
         query = model.User.search(q, query, user_name=context.get('user'))
@@ -861,7 +868,6 @@ def user_list(context, data_dict):
                     _or_(
                         model.Revision.author == model.User.name,
                         model.Revision.author == model.User.openid))))
-
     else:
         query = query.order_by(
             _case([(
@@ -879,9 +885,13 @@ def user_list(context, data_dict):
 
     users_list = []
 
-    for user in query.all():
-        result_dict = model_dictize.user_dictize(user[0], context)
-        users_list.append(result_dict)
+    if all_fields:
+        for user in query.all():
+            result_dict = model_dictize.user_dictize(user[0], context)
+            users_list.append(result_dict)
+    else:
+        for user in query.all():
+            users_list.append(user[0])
 
     return users_list
 

ckan/logic/auth/create.py

@@ -178,10 +178,8 @@ def _check_group_auth(context, data_dict):
     for group_blob in group_blobs:
         # group_blob might be a dict or a group_ref
         if isinstance(group_blob, dict):
-            if api_version == '1':
-                id = group_blob.get('name')
-            else:
-                id = group_blob.get('id')
+            # use group id by default, but we can accept name as well
+            id = group_blob.get('id') or group_blob.get('name')
             if not id:
                 continue
         else:

ckan/model/__init__.py

@@ -255,6 +255,8 @@ def delete_all(self):
         else:
             tables = reversed(self.metadata.sorted_tables)
         for table in tables:
+            if table.name == 'migrate_version':
+                continue
             connection.execute('delete from "%s"' % table.name)
         self.session.commit()
         log.info('Database table data deleted')

ckan/model/system_info.py

@@ -53,11 +53,15 @@ def __init__(self, key, value):
 
 def get_system_info(key, default=None):
     ''' get data from system_info table '''
-    obj = meta.Session.query(SystemInfo).filter_by(key=key).first()
-    if obj:
-        return obj.value
-    else:
-        return default
+    from sqlalchemy.exc import ProgrammingError
+    try:
+        obj = meta.Session.query(SystemInfo).filter_by(key=key).first()
+        if obj:
+            return obj.value
+    except ProgrammingError:
+        meta.Session.rollback()
+    return default
+
 
 
 def delete_system_info(key, default=None):

ckan/public/base/javascript/main.js

@@ -20,7 +20,6 @@ this.ckan = this.ckan || {};
   ckan.initialize = function () {
     var body = jQuery('body');
     var locale = jQuery('html').attr('lang');
-    var browserLocale = window.navigator.userLanguage || window.navigator.language;
     var location = window.location;
     var root = location.protocol + '//' + location.host;
 
@@ -33,7 +32,7 @@ this.ckan = this.ckan || {};
 
     // Convert all datetimes to the users timezone
     jQuery('.automatic-local-datetime').each(function() {
-        moment.locale(browserLocale);
+        moment.locale(locale);
         var date = moment(jQuery(this).data('datetime'));
         if (date.isValid()) {
             jQuery(this).html(date.format("LL, LT ([UTC]Z)")); 

ckan/templates/organization/member_new.html

@@ -56,15 +56,15 @@ <h1 class="page-heading">
     {{ form.select('role', label=_('Role'), options=c.roles, selected=c.user_role, error='', attrs=format_attrs) }}
     <div class="form-actions">
       {% if user %}
-        <a href="{% url_for controller='organization', action='member_delete', id=c.group_dict.id, user=user.id %}" class="btn btn-danger pull-left" data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this member?' }}">{{ _('Delete') }}</a>
+        <a href="{% url_for controller='organization', action='member_delete', id=c.group_dict.id, user=user.id %}" class="btn btn-danger pull-left" data-module="confirm-action" data-module-content="{{ _('Are you sure you want to delete this member?') }}">{{ _('Delete') }}</a>
         <button class="btn btn-primary" type="submit" name="submit" >
           {{ _('Update Member') }}
         </button>
       {% else %}
         <button class="btn btn-primary" type="submit" name="submit" >
           {{ _('Add Member') }}
         </button>
-      {% endif %}        
+      {% endif %}
     </div>
   </form>
   {% endblock %}

ckan/tests/legacy/functional/api/test_activity.py

@@ -21,6 +21,7 @@
 from nose import SkipTest
 from ckan.common import json
 import ckan.tests.legacy as tests
+from ckan.tests.helpers import call_action
 
 
 ##def package_update(context, data_dict):
@@ -190,6 +191,7 @@ def setup_class(self):
                 'name': sysadmin_user.name,
                 }
         normal_user = model.User.get('annafan')
+
         self.normal_user = {
                 'id': normal_user.id,
                 'apikey': normal_user.apikey,
@@ -276,7 +278,7 @@ def record_details(self, user_id, package_id=None, group_ids=None,
         details['recently changed datasets stream'] = \
                 self.recently_changed_datasets_stream(apikey)
 
-        details['time'] = datetime.datetime.now()
+        details['time'] = datetime.datetime.utcnow()
         return details
 
     def _create_package(self, user, name=None):
@@ -291,10 +293,17 @@ def _create_package(self, user, name=None):
 
         # Create a new package.
         request_data = make_package(name)
+
         before = self.record_details(user_id=user_id,
                 group_ids=[group['name'] for group in request_data['groups']],
                 apikey=apikey)
         extra_environ = {'Authorization': str(user['apikey'])}
+
+        call_action('member_create',
+                    capacity='admin',
+                    object=user['id'],
+                    object_type='user',
+                    id='roger')
         response = self.app.post('/api/action/package_create',
                 json.dumps(request_data), extra_environ=extra_environ)
         response_dict = json.loads(response.body)
@@ -1404,7 +1413,7 @@ def test_create_user(self):
         a new user is created.
 
         """
-        before = datetime.datetime.now()
+        before = datetime.datetime.utcnow()
 
         # Create a new user.
         user_dict = {'name': 'testuser',

ckan/tests/lib/test_helpers.py

@@ -205,6 +205,45 @@ def test_render_naughty_markdown(self):
         output = u'<ul>\n<li>[Foo (<a href="http://foo.bar" target="_blank" rel="nofollow">http://foo.bar</a>) * Bar] (<a href="http://foo.bar" target="_blank" rel="nofollow">http://foo.bar</a>)</li>\n</ul>'
         eq_(h.render_markdown(data), output)
 
+    def test_render_markdown_with_js(self):
+        data = u'[text](javascript: alert(1))'
+        output = u'<p><a>text</a></p>'
+        eq_(h.render_markdown(data), output)
+
+    def test_event_attributes(self):
+        data = u'<p onclick="some.script"><img onmouseover="some.script" src="image.png" /> and text</p>'
+        output = u'<p>and text</p>'
+        eq_(h.render_markdown(data), output)
+
+    def test_ampersand_in_links(self):
+        data = u'[link](/url?a=1&b=2)'
+        output = u'<p><a href="/url?a=1&amp;b=2">link</a></p>'
+        eq_(h.render_markdown(data), output)
+
+        data = u'http://example.com/page?a=1&b=2'
+        output = u'<p><a href="http://example.com/page?a=1&amp;b=2" target="_blank" rel="nofollow">http://example.com/page?a=1&amp;b=2</a></p>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_h1(self):
+        data = u'#heading'
+        output = u'<h1>heading</h1>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_h2(self):
+        data = u'##heading'
+        output = u'<h2>heading</h2>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_h3(self):
+        data = u'###heading'
+        output = u'<h3>heading</h3>'
+        eq_(h.render_markdown(data), output)
+
+    def test_tags_img(self):
+        data = u'![image](/image.png)'
+        output = u'<p><img alt="image" src="/image.png"></p>'
+        eq_(h.render_markdown(data), output)
+
 
 class TestHelpersRemoveLineBreaks(object):
 

ckan/tests/lib/test_jobs.py

@@ -7,6 +7,7 @@
 import datetime
 
 from nose.tools import ok_, assert_equal, raises
+from nose.plugins.skip import SkipTest
 import rq
 
 import ckan.lib.jobs as jobs
@@ -243,6 +244,7 @@ def test_fork_within_a_transaction(self):
         The horse should get a new SQLAlchemy session but leave the
         original session alone.
         '''
+        raise SkipTest(u'Failing intermittently')  # FIXME
         pkg_name = u'test-fork-within-a-transaction'
         model.repo.new_revision()
         pkg = model.Package.get(pkg_name)