diff --git a/.gitignore b/.gitignore index daf2c2e2007..b765bfee61f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,8 +6,6 @@ syntax: glob .DS_Store ckan.egg-info/* sandbox/* -theme/* -theme dist # pylons @@ -28,9 +26,6 @@ fl_notes.txt .noseids *~ -# local symlinks -ckan/public/scripts/ckanjs.js - # custom style ckan/public/base/less/custom.less diff --git a/README.rst b/README.rst index fce4eeea262..17fce2e20d0 100644 --- a/README.rst +++ b/README.rst @@ -23,19 +23,16 @@ Community * Developer mailing list: `ckan-dev@lists.okfn.org `_ * Developer IRC channel: `#ckan on irc.freenode.net `_ -* Issue tracker: `trac.ckan.org `_ +* `Issue tracker `_ * `CKAN tag on StackOverflow `_ +* `Wiki `_ Contributing to CKAN -------------------- -CKAN is a free software project and code contributions are welcome. -The `For CKAN Developers `_ -section of the documentation explains how to contribute to CKAN or its documentation, -including our **coding standards**. - -The `CKAN Wiki `_ is also open fo contributions. +For contributing to CKAN or its documentation, see +`CONTRIBUTING `_. Copying and License diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index cf22ba9ad78..f13e8eb7e14 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -3,6 +3,7 @@ import cgi import datetime import glob +import urllib from pylons import c, request, response from pylons.i18n import _, gettext @@ -545,7 +546,7 @@ def search(self, ver=None, register=None): def _get_search_params(cls, request_params): if 'qjson' in request_params: try: - qjson_param = request_params['qjson'].replace('\\\\u','\\u') + qjson_param = request_params['qjson'].replace('\\\\u', '\\u') params = h.json.loads(qjson_param, encoding='utf8') except ValueError, e: raise ValueError(gettext('Malformed qjson value') + ': %r' @@ -668,9 +669,8 @@ def group_exists(val): def dataset_autocomplete(self): q = request.params.get('incomplete', '') - q_lower = q.lower() limit = request.params.get('limit', 10) - tag_names = [] + package_dicts = [] if q: context = {'model': model, 'session': model.Session, 'user': c.user or c.author} @@ -752,9 +752,103 @@ def i18n_js_translations(self, lang): ''' translation strings for front end ''' ckan_path = os.path.join(os.path.dirname(__file__), '..') source = os.path.abspath(os.path.join(ckan_path, 'public', - 'base', 'i18n', '%s.js' % lang)) + 'base', 'i18n', '%s.js' % lang)) response.headers['Content-Type'] = CONTENT_TYPES['json'] if not os.path.exists(source): return '{}' f = open(source, 'r') return(f) + + @classmethod + def _get_request_data(cls, try_url_params=False): + '''Returns a dictionary, extracted from a request. + + If there is no data, None or "" is returned. + ValueError will be raised if the data is not a JSON-formatted dict. + + The data is retrieved as a JSON-encoded dictionary from the request + body. Or, if the `try_url_params` argument is True and the request is + a GET request, then an attempt is made to read the data from the url + parameters of the request. + + try_url_params + If try_url_params is False, then the data_dict is read from the + request body. + + If try_url_params is True and the request is a GET request then the + data is read from the url parameters. The resulting dict will only + be 1 level deep, with the url-param fields being the keys. If a + single key has more than one value specified, then the value will + be a list of strings, otherwise just a string. + + ''' + def make_unicode(entity): + '''Cast bare strings and strings in lists or dicts to Unicode. ''' + if isinstance(entity, str): + return unicode(entity) + elif isinstance(entity, list): + new_items = [] + for item in entity: + new_items.append(make_unicode(item)) + return new_items + elif isinstance(entity, dict): + new_dict = {} + for key, val in entity.items(): + new_dict[key] = make_unicode(val) + return new_dict + else: + return entity + + cls.log.debug('Retrieving request params: %r' % request.params) + cls.log.debug('Retrieving request POST: %r' % request.POST) + cls.log.debug('Retrieving request GET: %r' % request.GET) + request_data = None + if request.POST: + try: + keys = request.POST.keys() + # Parsing breaks if there is a = in the value, so for now + # we will check if the data is actually all in a single key + if keys and request.POST[keys[0]] in [u'1', u'']: + request_data = keys[0] + else: + request_data = urllib.unquote_plus(request.body) + except Exception, inst: + msg = "Could not find the POST data: %r : %s" % \ + (request.POST, inst) + raise ValueError(msg) + + elif try_url_params and request.GET: + return request.GET.mixed() + + else: + try: + if request.method in ['POST', 'PUT']: + request_data = request.body + else: + request_data = None + except Exception, inst: + msg = "Could not extract request body data: %s" % \ + (inst) + raise ValueError(msg) + cls.log.debug('Retrieved request body: %r' % request.body) + if not request_data: + msg = "No request body data" + raise ValueError(msg) + if request_data: + try: + request_data = h.json.loads(request_data, encoding='utf8') + except ValueError, e: + raise ValueError('Error decoding JSON data. ' + 'Error: %r ' + 'JSON data extracted from the request: %r' % + (e, request_data)) + if not isinstance(request_data, dict): + raise ValueError('Request data JSON decoded to %r but ' + 'it needs to be a dictionary.' % request_data) + # ensure unicode values + for key, val in request_data.items(): + # if val is str then assume it is ascii, since json converts + # utf8 encoded JSON to unicode + request_data[key] = make_unicode(val) + cls.log.debug('Request data extracted: %r' % request_data) + return request_data diff --git a/ckan/controllers/group.py b/ckan/controllers/group.py index bbcb4f7bf64..4e9e2ea5585 100644 --- a/ckan/controllers/group.py +++ b/ckan/controllers/group.py @@ -672,7 +672,11 @@ def followers(self, id): context = {'model': model, 'session': model.Session, 'user': c.user or c.author} c.group_dict = self._get_group_dict(id) - c.followers = get_action('group_follower_list')(context, {'id': id}) + try: + c.followers = get_action('group_follower_list')(context, + {'id': id}) + except NotAuthorized: + abort(401, _('Unauthorized to view followers %s') % '') return render('group/followers.html') def admins(self, id): diff --git a/ckan/controllers/home.py b/ckan/controllers/home.py index 1fe85d47b56..a766771cd39 100644 --- a/ckan/controllers/home.py +++ b/ckan/controllers/home.py @@ -125,6 +125,7 @@ def db_to_form_schema(group_type=None): 'ignore_auth': True, 'user': c.user or c.author, 'schema': db_to_form_schema(group_type=group_type), + 'limits': {'packages': 2}, 'for_view': True} data_dict = {'id': id} diff --git a/ckan/controllers/revision.py b/ckan/controllers/revision.py index b3e6a5dfdf9..84683c1efc9 100644 --- a/ckan/controllers/revision.py +++ b/ckan/controllers/revision.py @@ -69,6 +69,8 @@ def list(self): # but in the meantime while that is fixed, # avoid an exception here continue + if package.private: + continue number = len(package.all_revisions) package_revision = None count = 0 @@ -142,10 +144,11 @@ def read(self, id=None): pkgs = model.Session.query(model.PackageRevision).\ filter_by(revision=c.revision) - c.packages = [pkg.continuity for pkg in pkgs] + c.packages = [pkg.continuity for pkg in pkgs if not pkg.private] pkgtags = model.Session.query(model.PackageTagRevision).\ filter_by(revision=c.revision) - c.pkgtags = [pkgtag.continuity for pkgtag in pkgtags] + c.pkgtags = [pkgtag.continuity for pkgtag in pkgtags + if not pkgtag.package.private] grps = model.Session.query(model.GroupRevision).\ filter_by(revision=c.revision) c.groups = [grp.continuity for grp in grps] diff --git a/ckan/controllers/user.py b/ckan/controllers/user.py index af3ad349935..b0e13dccd2c 100644 --- a/ckan/controllers/user.py +++ b/ckan/controllers/user.py @@ -479,7 +479,10 @@ def followers(self, id=None): data_dict = {'id': id, 'user_obj': c.userobj} self._setup_template_variables(context, data_dict) f = get_action('user_follower_list') - c.followers = f(context, {'id': c.user_dict['id']}) + try: + c.followers = f(context, {'id': c.user_dict['id']}) + except NotAuthorized: + abort(401, _('Unauthorized to view followers %s') % '') return render('user/followers.html') def activity(self, id, offset=0): diff --git a/ckan/lib/activity_streams_session_extension.py b/ckan/lib/activity_streams_session_extension.py index 9fbc1a67e7f..08a6e37756c 100644 --- a/ckan/lib/activity_streams_session_extension.py +++ b/ckan/lib/activity_streams_session_extension.py @@ -79,6 +79,11 @@ def before_commit(self, session): # object is a package. logger.debug("Looks like this object is a package") logger.debug("activity: %s" % activity) + + # Don't create activities for private datasets. + if obj.private: + continue + activities[obj.id] = activity activity_detail = activity_stream_detail(obj, activity.id, "new") @@ -114,6 +119,10 @@ def before_commit(self, session): for package in related_packages: if package is None: continue + # Don't create activities for private datasets. + if package.private: + continue + if package.id in activities: activity = activities[package.id] else: diff --git a/ckan/lib/base.py b/ckan/lib/base.py index 4fd2953ed21..33253d6e0d0 100644 --- a/ckan/lib/base.py +++ b/ckan/lib/base.py @@ -86,7 +86,7 @@ def render_jinja2(template_name, extra_vars): def render(template_name, extra_vars=None, cache_key=None, cache_type=None, cache_expire=None, method='xhtml', loader_class=MarkupTemplate, - cache_force = None, renderer=None): + cache_force=None, renderer=None): ''' Main template rendering function. ''' def render_template(): @@ -101,12 +101,12 @@ def render_template(): try: template_path, template_type = lib.render.template_info(template_name) except lib.render.TemplateNotFound: - template_type = 'genshi' + template_type = 'genshi' template_path = '' # snippets should not pass the context # but allow for legacy genshi templates - if renderer == 'snippet' and template_type != 'genshi': + if renderer == 'snippet' and template_type != 'genshi': del globs['c'] del globs['tmpl_context'] @@ -115,12 +115,12 @@ def render_template(): context_vars = globs.get('c') if context_vars: context_vars = dir(context_vars) - debug_info = {'template_name' : template_name, - 'template_path' : template_path, - 'template_type' : template_type, - 'vars' : globs, + debug_info = {'template_name': template_name, + 'template_path': template_path, + 'template_type': template_type, + 'vars': globs, 'c_vars': context_vars, - 'renderer' : renderer,} + 'renderer': renderer} if 'CKAN_DEBUG_INFO' not in request.environ: request.environ['CKAN_DEBUG_INFO'] = [] request.environ['CKAN_DEBUG_INFO'].append(debug_info) @@ -221,9 +221,9 @@ def __before__(self, action, **params): if c.userobj: from ckan.logic import get_action new_activities_count = get_action( - 'dashboard_new_activities_count') + 'dashboard_new_activities_count') context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} + 'user': c.user or c.author} c.new_activities = new_activities_count(context, {}) def _identify_user(self): @@ -340,119 +340,6 @@ def _set_cors(self): response.headers['Access-Control-Allow-Headers'] = \ "X-CKAN-API-KEY, Authorization, Content-Type" - def _get_user(self, reference): - return model.User.by_name(reference) - - def _get_pkg(self, reference): - return model.Package.get(reference) - - def _get_group(self, reference): - return model.Group.get(reference) - - def _get_tag(self, reference): - return model.Tag.get(reference) - - @classmethod - def _get_request_data(cls, try_url_params=False): - '''Returns a dictionary, extracted from a request. - - If there is no data, None or "" is returned. - ValueError will be raised if the data is not a JSON-formatted dict. - - The data is retrieved as a JSON-encoded dictionary from the request - body. Or, if the `try_url_params` argument is True and the request is - a GET request, then an attempt is made to read the data from the url - parameters of the request. - - try_url_params - If try_url_params is False, then the data_dict is read from the - request body. - - If try_url_params is True and the request is a GET request then the - data is read from the url parameters. The resulting dict will only - be 1 level deep, with the url-param fields being the keys. If a - single key has more than one value specified, then the value will - be a list of strings, otherwise just a string. - - This function is only used by the API, so no strings need to be - translated. - - TODO: If this is only used by the API, then perhaps it should be - moved to the api controller class? - ''' - cls.log.debug('Retrieving request params: %r' % request.params) - cls.log.debug('Retrieving request POST: %r' % request.POST) - cls.log.debug('Retrieving request GET: %r' % request.GET) - request_data = None - if request.POST: - try: - keys = request.POST.keys() - # Parsing breaks if there is a = in the value, so for now - # we will check if the data is actually all in a single key - if keys and request.POST[keys[0]] in [u'1', u'']: - request_data = keys[0] - else: - request_data = urllib.unquote_plus(request.body) - except Exception, inst: - msg = "Could not find the POST data: %r : %s" % \ - (request.POST, inst) - raise ValueError(msg) - - elif try_url_params and request.GET: - return request.GET.mixed() - - else: - try: - if request.method in ['POST', 'PUT']: - request_data = request.body - else: - request_data = None - except Exception, inst: - msg = "Could not extract request body data: %s" % \ - (inst) - raise ValueError(msg) - cls.log.debug('Retrieved request body: %r' % request.body) - if not request_data: - msg = "No request body data" - raise ValueError(msg) - if request_data: - try: - request_data = json.loads(request_data, encoding='utf8') - except ValueError, e: - raise ValueError('Error decoding JSON data. ' - 'Error: %r ' - 'JSON data extracted from the request: %r' % - (e, request_data)) - if not isinstance(request_data, dict): - raise ValueError('Request data JSON decoded to %r but ' - 'it needs to be a dictionary.' % request_data) - # ensure unicode values - for key, val in request_data.items(): - # if val is str then assume it is ascii, since json converts - # utf8 encoded JSON to unicode - request_data[key] = cls._make_unicode(val) - cls.log.debug('Request data extracted: %r' % request_data) - return request_data - - @classmethod - def _make_unicode(cls, entity): - """Cast bare strings and strings in lists or dicts to Unicode - """ - if isinstance(entity, str): - return unicode(entity) - elif isinstance(entity, list): - new_items = [] - for item in entity: - new_items.append(cls._make_unicode(item)) - return new_items - elif isinstance(entity, dict): - new_dict = {} - for key, val in entity.items(): - new_dict[key] = cls._make_unicode(val) - return new_dict - else: - return entity - def _get_user_for_apikey(self): apikey_header_name = config.get(APIKEY_HEADER_NAME_KEY, APIKEY_HEADER_NAME_DEFAULT) @@ -475,115 +362,6 @@ def _get_user_for_apikey(self): user = query.filter_by(apikey=apikey).first() return user - def _get_timing_cache_path(self): - - return path - - def _handle_update_of_authz(self, domain_object): - '''In the event of a post request to a domain object\'s authz form, - work out which of the four possible actions is to be done, - and do it before displaying the page. - - Returns the updated roles for the domain_object. - ''' - from ckan.logic import NotFound, get_action - - context = {'model': model, 'session': model.Session, - 'user': c.user or c.author} - data_dict = {'domain_object': domain_object.id} - - # Work out actions needed, depending on which button was pressed - update_type = 'user' - if 'save' in request.POST: - update_or_add = 'update' - elif 'add' in request.POST: - update_or_add = 'add' - else: - update_type = None - update_or_add = None - - # Work out what role checkboxes are checked or unchecked - checked_roles = [box_id for (box_id, value) in request.params.items() - if (value == u'on')] - unchecked_roles = [box_id for (box_id, value) in request.params.items() - if (value == u'submitted')] - - action = None - if update_or_add is 'update': - # Get user_roles by decoding the checkbox grid - user$role strings - user_roles = {} - for checked_role in checked_roles: - obj_id, role = checked_role.split('$') - if obj_id not in user_roles: - user_roles[obj_id] = [] - user_roles[obj_id].append(role) - # Users without roles need adding to the user_roles too to make - # their roles be deleted - for unchecked_role in unchecked_roles: - obj_id, role = unchecked_role.split('$') - if obj_id not in user_roles: - user_roles[obj_id] = [] - # Convert user_roles to role dictionaries - role_dicts = [] - for user, roles in user_roles.items(): - role_dicts.append({update_type: user, 'roles': roles}) - data_dict['user_roles'] = role_dicts - - action = 'user_role_bulk_update' - success_message = _('Updated') - elif update_or_add is 'add': - # Roles for this new user is a simple list from the checkbox row - data_dict['roles'] = checked_roles - - # User comes from the input box. - new_user = request.params.get('new_user_name') - if new_user: - data_dict[update_type] = new_user - - action = 'user_role_update' - success_message = _('User role(s) added') - else: - h.flash_error(_('Please supply a user name')) - - if action: - try: - roles = get_action(action)(context, data_dict) - except NotFound, e: - h.flash_error(_('Not found') + (': %s' % e if str(e) else '')) - else: - h.flash_success(success_message) - - # Return roles for all users on this domain object - if update_or_add is 'add': - if update_type in data_dict: - del data_dict[update_type] - return get_action('roles_show')(context, data_dict) - - def _prepare_authz_info_for_render(self, user_object_roles): - # ================= - # Display the page - - # Find out all the possible roles. At the moment, any role can be - # associated with any object, so that's easy: - possible_roles = model.Role.get_all() - - # uniquify and sort - users = sorted(list(set([uor['user_id'] - for uor in user_object_roles['roles'] - if uor['user_id']]))) - - # make a dictionary from (user, role) to True, False - users_roles = [(uor['user_id'], uor['role']) - for uor in user_object_roles['roles'] - if uor['user_id']] - user_role_dict = {} - for u in users: - for r in possible_roles: - user_role_dict[(u, r)] = (u, r) in users_roles - - c.roles = possible_roles - c.users = users - c.user_role_dict = user_role_dict # Include the '_' function in the public names __all__ = [__name for __name in locals().keys() if not __name.startswith('_') diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index 40cc566624d..490b5601fc5 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -9,6 +9,7 @@ import ckan.lib.helpers as h import ckan.lib.dictization as d import ckan.new_authz as new_authz +import ckan.lib.search as search ## package save @@ -17,6 +18,15 @@ def group_list_dictize(obj_list, context, active = context.get('active', True) with_private = context.get('include_private_packages', False) + + query = search.PackageSearchQuery() + + q = {'q': '+capacity:public' if not with_private else '*:*', + 'fl': 'groups', 'facet.field': ['groups'], + 'facet.limit': -1, 'rows': 1} + + query.run(q) + result_list = [] for obj in obj_list: @@ -31,8 +41,7 @@ def group_list_dictize(obj_list, context, group_dict['display_name'] = obj.display_name - group_dict['packages'] = \ - len(obj.packages(with_private=with_private, context=context)) + group_dict['packages'] = query.facets['groups'].get(obj.name, 0) if context.get('for_view'): if group_dict['is_organization']: @@ -302,11 +311,17 @@ def _get_members(context, group, member_type): model = context['model'] Entity = getattr(model, member_type[:-1].capitalize()) - return model.Session.query(Entity, model.Member.capacity).\ + q = model.Session.query(Entity, model.Member.capacity).\ join(model.Member, model.Member.table_id == Entity.id).\ filter(model.Member.group_id == group.id).\ filter(model.Member.state == 'active').\ - filter(model.Member.table_name == member_type[:-1]).all() + filter(model.Member.table_name == member_type[:-1]) + if member_type == 'packages': + q = q.filter(Entity.private==False) + if 'limits' in context and member_type in context['limits']: + return q[:context['limits'][member_type]] + return q.all() + def group_dictize(group, context): model = context['model'] @@ -323,6 +338,10 @@ def group_dictize(group, context): _get_members(context, group, 'packages'), context) + query = search.PackageSearchQuery() + q = {'q': 'groups:"%s" +capacity:public' % group.name, 'rows': 1} + result_dict['package_count'] = query.run(q)['count'] + result_dict['tags'] = tag_list_dictize( _get_members(context, group, 'tags'), context) @@ -376,8 +395,14 @@ def tag_list_dictize(tag_list, context): def tag_dictize(tag, context): result_dict = d.table_dictize(tag, context) - result_dict["packages"] = d.obj_list_dictize(tag.packages, context) + query = search.PackageSearchQuery() + + q = {'q': '+tags:"%s" +capacity:public' % tag.name, 'fl': 'data_dict', + 'wt': 'json', 'rows': 1000} + result_dict["packages"] = [ + h.json.loads(result['data_dict']) for result in query.run(q)['results'] + ] # Add display_names to tags. At first a tag's display_name is just the # same as its name, but the display_name might get changed later (e.g. # translated into another language by the multilingual extension). diff --git a/ckan/lib/helpers.py b/ckan/lib/helpers.py index 70aa3e88f35..3f32d66d905 100644 --- a/ckan/lib/helpers.py +++ b/ckan/lib/helpers.py @@ -322,11 +322,11 @@ def _create_link_text(text, **kwargs): kwargs ''' if kwargs.pop('inner_span', None): text = literal('') + text + literal('') - icon = kwargs.pop('icon', None) if icon: text = literal(' ' % icon) + text return text + icon = kwargs.pop('icon', None) class_ = _link_class(kwargs) return link_to( _create_link_text(text, **kwargs), @@ -568,6 +568,8 @@ def check_access(action, data_dict=None): context = {'model': model, 'user': c.user or c.author} + if not data_dict: + data_dict = {} try: check_access_logic(action, context, data_dict) diff --git a/ckan/logic/action/create.py b/ckan/logic/action/create.py index b9f1aca8679..13495a0d4df 100644 --- a/ckan/logic/action/create.py +++ b/ckan/logic/action/create.py @@ -1028,24 +1028,6 @@ def follow_user(context, data_dict): follower = model_save.follower_dict_save(validated_data_dict, context, model.UserFollowingUser) - activity_dict = { - 'user_id': userobj.id, - 'object_id': validated_data_dict['id'], - 'activity_type': 'follow user', - } - activity_dict['data'] = { - 'user': ckan.lib.dictization.table_dictize( - model.User.get(validated_data_dict['id']), context), - } - activity_create_context = { - 'model': model, - 'user': userobj, - 'defer_commit': True, - 'session': session - } - logic.get_action('activity_create')(activity_create_context, - activity_dict, ignore_auth=True) - if not context.get('defer_commit'): model.repo.commit() @@ -1099,24 +1081,6 @@ def follow_dataset(context, data_dict): follower = model_save.follower_dict_save(validated_data_dict, context, model.UserFollowingDataset) - activity_dict = { - 'user_id': userobj.id, - 'object_id': validated_data_dict['id'], - 'activity_type': 'follow dataset', - } - activity_dict['data'] = { - 'dataset': ckan.lib.dictization.table_dictize( - model.Package.get(validated_data_dict['id']), context), - } - activity_create_context = { - 'model': model, - 'user': userobj, - 'defer_commit':True, - 'session': session - } - logic.get_action('activity_create')(activity_create_context, - activity_dict, ignore_auth=True) - if not context.get('defer_commit'): model.repo.commit() @@ -1209,24 +1173,6 @@ def follow_group(context, data_dict): follower = model_save.follower_dict_save(validated_data_dict, context, model.UserFollowingGroup) - activity_dict = { - 'user_id': userobj.id, - 'object_id': validated_data_dict['id'], - 'activity_type': 'follow group', - } - activity_dict['data'] = { - 'group': ckan.lib.dictization.table_dictize( - model.Group.get(validated_data_dict['id']), context), - } - activity_create_context = { - 'model': model, - 'user': userobj, - 'defer_commit': True, - 'session': session - } - logic.get_action('activity_create')(activity_create_context, - activity_dict, ignore_auth=True) - if not context.get('defer_commit'): model.repo.commit() diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index a1314caf458..983e530ea12 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -827,6 +827,7 @@ def _group_or_org_show(context, data_dict, is_org=False): _check_access('organization_show',context, data_dict) else: _check_access('group_show',context, data_dict) + group_dict = model_dictize.group_dictize(group, context) @@ -934,16 +935,7 @@ def tag_show(context, data_dict): _check_access('tag_show',context, data_dict) - tag_dict = model_dictize.tag_dictize(tag,context) - - extended_packages = [] - for package in tag_dict['packages']: - pkg = model.Package.get(package['id']) - extended_packages.append(model_dictize.package_dictize(pkg,context)) - - tag_dict['packages'] = extended_packages - - return tag_dict + return model_dictize.tag_dictize(tag,context) def user_show(context, data_dict): '''Return a user account. @@ -2245,6 +2237,7 @@ def user_follower_list(context, data_dict): :rtype: list of dictionaries ''' + _check_access('user_follower_list', context, data_dict) return _follower_list(context, data_dict, ckan.logic.schema.default_follow_user_schema(), context['model'].UserFollowingUser) @@ -2259,6 +2252,7 @@ def dataset_follower_list(context, data_dict): :rtype: list of dictionaries ''' + _check_access('dataset_follower_list', context, data_dict) return _follower_list(context, data_dict, ckan.logic.schema.default_follow_dataset_schema(), context['model'].UserFollowingDataset) @@ -2273,6 +2267,7 @@ def group_follower_list(context, data_dict): :rtype: list of dictionaries ''' + _check_access('group_follower_list', context, data_dict) return _follower_list(context, data_dict, ckan.logic.schema.default_follow_group_schema(), context['model'].UserFollowingGroup) @@ -2438,6 +2433,7 @@ def followee_list(context, data_dict): returned by user_show, package_show or group_show) ''' + _check_access('followee_list', context, data_dict) schema = context.get('schema') or ( ckan.logic.schema.default_follow_user_schema()) data_dict, errors = _validate(data_dict, schema, context) @@ -2456,6 +2452,7 @@ def display_name(followee): # TODO: Catch exceptions raised by these *_followee_list() functions? followee_dicts = [] context['skip_validation'] = True + context['skip_authorization'] = True for followee_list_function, followee_type in ( (user_followee_list, 'user'), (dataset_followee_list, 'dataset'), @@ -2490,6 +2487,9 @@ def user_followee_list(context, data_dict): :rtype: list of dictionaries ''' + if not context.get('skip_authorization'): + _check_access('user_followee_list', context, data_dict) + if not context.get('skip_validation'): schema = context.get('schema') or ( ckan.logic.schema.default_follow_user_schema()) @@ -2518,6 +2518,9 @@ def dataset_followee_list(context, data_dict): :rtype: list of dictionaries ''' + if not context.get('skip_authorization'): + _check_access('dataset_followee_list', context, data_dict) + if not context.get('skip_validation'): schema = context.get('schema') or ( ckan.logic.schema.default_follow_user_schema()) @@ -2547,6 +2550,9 @@ def group_followee_list(context, data_dict): :rtype: list of dictionaries ''' + if not context.get('skip_authorization'): + _check_access('group_followee_list', context, data_dict) + if not context.get('skip_validation'): schema = context.get('schema', ckan.logic.schema.default_follow_user_schema()) diff --git a/ckan/logic/auth/get.py b/ckan/logic/auth/get.py index 95a75b0d1af..3df5689c2ea 100644 --- a/ckan/logic/auth/get.py +++ b/ckan/logic/auth/get.py @@ -217,3 +217,48 @@ def dashboard_new_activities_count(context, data_dict): # This is so a better not authourized message can be sent. return new_authz.is_authorized('dashboard_activity_list', context, data_dict) + + +def user_follower_list(context, data_dict): + return sysadmin(context, data_dict) + + +def dataset_follower_list(context, data_dict): + return sysadmin(context, data_dict) + + +def group_follower_list(context, data_dict): + return sysadmin(context, data_dict) + + +def _followee_list(context, data_dict): + model = context['model'] + + # Visitors cannot see what users are following. + authorized_user = model.User.get(context.get('user')) + if not authorized_user: + return {'success': False, 'msg': _('Not authorized')} + + # Any user is authorized to see what she herself is following. + requested_user = model.User.get(data_dict.get('id')) + if authorized_user == requested_user: + return {'success': True} + + # Sysadmins are authorized to see what anyone is following. + return sysadmin(context, data_dict) + + +def followee_list(context, data_dict): + return _followee_list(context, data_dict) + + +def user_followee_list(context, data_dict): + return _followee_list(context, data_dict) + + +def dataset_followee_list(context, data_dict): + return _followee_list(context, data_dict) + + +def group_followee_list(context, data_dict): + return _followee_list(context, data_dict) diff --git a/ckan/model/__init__.py b/ckan/model/__init__.py index 94d30d9c0be..7b3d7e5887a 100644 --- a/ckan/model/__init__.py +++ b/ckan/model/__init__.py @@ -13,6 +13,7 @@ from meta import ( Session, engine_is_sqlite, + engine_is_pg, ) from core import ( System, @@ -460,7 +461,8 @@ def revision_as_dict(revision, include_packages=True, include_groups=True, )) if include_packages: revision_dict['packages'] = [getattr(pkg, ref_package_by) \ - for pkg in revision.packages if pkg] + for pkg in revision.packages + if (pkg and not pkg.private)] if include_groups: revision_dict['groups'] = [getattr(grp, ref_package_by) \ for grp in revision.groups if grp] diff --git a/ckan/model/meta.py b/ckan/model/meta.py index 41d67541a1b..ce175d309d7 100644 --- a/ckan/model/meta.py +++ b/ckan/model/meta.py @@ -10,7 +10,7 @@ import extension import ckan.lib.activity_streams_session_extension as activity -__all__ = ['Session', 'engine_is_sqlite'] +__all__ = ['Session', 'engine_is_sqlite', 'engine_is_pg'] class CkanCacheExtension(SessionExtension): @@ -153,8 +153,14 @@ def after_rollback(self, session): # names, you'll need a metadata for each database metadata = MetaData() + def engine_is_sqlite(): - """ - Returns true iff the engine is connected to a sqlite database. - """ + # Returns true iff the engine is connected to a sqlite database. return engine.url.drivername == 'sqlite' + + +def engine_is_pg(): + # Returns true iff the engine is connected to a postgresql database. + # According to http://docs.sqlalchemy.org/en/latest/core/engines.html#postgresql + # all Postgres driver names start with `postgresql` + return engine.url.drivername.startswith('postgresql') diff --git a/ckan/public/base/css/main.css b/ckan/public/base/css/main.css index 272bd415d7b..66aab31b86b 100644 --- a/ckan/public/base/css/main.css +++ b/ckan/public/base/css/main.css @@ -4506,13 +4506,13 @@ ul.icons li .icon-large:before { .simple-list:after { clear: both; } -.simple-list > li { +.simple-list > li { font-size: 12px; line-height: 1.1666666666666667em; padding: 7px 25px; border-bottom: 1px dotted #cccccc; } -.simple-list > li:last-of-type { +.simple-list > li:last-of-type { border-bottom: 0; } .simple-list .ckan-icon { @@ -4669,8 +4669,6 @@ ul.icons li .icon-large:before { padding-right: 15px; } .module-grid { - margin: 0; - list-style: none; margin: 0; list-style: none; *zoom: 1; @@ -4694,7 +4692,6 @@ ul.icons li .icon-large:before { padding-left: 20px; padding-bottom: 25px; float: left; - float: left; margin-left: 20px; width: 460px; padding-top: 10px; @@ -4720,10 +4717,10 @@ ul.icons li .icon-large:before { .ckanext-datapreview { position: relative; } -.ckanext-datapreview > iframe { +.ckanext-datapreview > iframe { min-height: 400px; } -.ckanext-datapreview > img { +.ckanext-datapreview > img { max-height: 500px; max-width: 100%; overflow: hidden; @@ -4887,13 +4884,13 @@ ol.media-grid:after { .nav-simple:after { clear: both; } -.nav-simple > li { +.nav-simple > li { font-size: 12px; line-height: 1.1666666666666667em; padding: 7px 25px; border-bottom: 1px dotted #cccccc; } -.nav-simple > li:last-of-type { +.nav-simple > li:last-of-type { border-bottom: 0; } .nav-simple .ckan-icon { @@ -4915,12 +4912,10 @@ ol.media-grid:after { } .nav-item.active > a { background: url("../../../base/images/background-tag.png") no-repeat -13px center; - position: relative; display: block; font-size: 11px; line-height: 27px; color: #187794; - padding-left: 10px; padding-right: 5px; margin-right: 11px; -webkit-box-sizing: border-box; @@ -5226,8 +5221,6 @@ textarea { background-image: url("../../../base/images/sprite-ckan-icons.png"); background-repeat: no-repeat; background-position: 16px 16px; - width: 17px; - height: 17px; background-position: -51px -16px; position: absolute; display: block; @@ -5654,6 +5647,13 @@ textarea { .control-full .select2-container { width: 520px !important; } +.control-group.error .select2-container input:focus, +.control-group.error .select2-container select:focus, +.control-group.error .select2-container textarea:focus { + -webkit-box-shadow: none; + -moz-box-shadow: none; + box-shadow: none; +} .dataset-item { border-bottom: 1px dotted #cccccc; padding-bottom: 20px; @@ -5821,8 +5821,6 @@ textarea { vertical-align: text-bottom; position: relative; top: 2px; - width: 16px; - height: 16px; background-image: url("../../../base/images/sprite-ckan-icons.png"); background-repeat: no-repeat; background-position: 16px 16px; @@ -6682,7 +6680,6 @@ li .icon-large:before { float: right; } [role=main] .secondary { - float: left; margin-left: 20px; width: 220px; margin-left: 0; diff --git a/ckan/public/base/less/forms.less b/ckan/public/base/less/forms.less index 445c2677ec2..5902a859f8b 100644 --- a/ckan/public/base/less/forms.less +++ b/ckan/public/base/less/forms.less @@ -686,3 +686,11 @@ textarea { // sets it on the element. width: 520px !important; } + +.control-group.error .select2-container { + input:focus, + select:focus, + textarea:focus { + .box-shadow(none); + } +} diff --git a/ckan/templates/admin/authz.html b/ckan/templates/admin/authz.html index cecb18a9f75..5b60f95cc27 100644 --- a/ckan/templates/admin/authz.html +++ b/ckan/templates/admin/authz.html @@ -14,9 +14,9 @@ {% endfor %} - {{ _('User') }} + {{ _('User') }} {% for role in roles %} - {{ role }} + {{ role }} {% endfor %} {% for user in users %} @@ -49,9 +49,9 @@ {% endfor %} - {{ _('User') }} + {{ _('User') }} {% for role in roles %} - {{ role }} + {{ role }} {% endfor %} @@ -77,9 +77,9 @@ {% endfor %} - {{ _('User Group') }} + {{ _('User Group') }} {% for role in roles %} - {{ role }} + {{ role }} {% endfor %} {% for user in users %} @@ -112,9 +112,9 @@ {% endfor %} - User Group + User Group {% for role in roles %} - {{ role }} + {{ role }} {% endfor %} diff --git a/ckan/templates/ajax_snippets/api_info.html b/ckan/templates/ajax_snippets/api_info.html index f339054e86d..20fc44281b8 100644 --- a/ckan/templates/ajax_snippets/api_info.html +++ b/ckan/templates/ajax_snippets/api_info.html @@ -39,19 +39,19 @@

- Create + Create {{ datastore_root_url }}/datastore_create - Update / Insert + Update / Insert {{ datastore_root_url }}/datastore_upsert - Query + Query {{ datastore_root_url }}/datastore_search - Query (via SQL) + Query (via SQL) {{ datastore_root_url }}/datastore_search_sql diff --git a/ckan/templates/group/read_base.html b/ckan/templates/group/read_base.html index f476bce7a6b..a16615b7777 100644 --- a/ckan/templates/group/read_base.html +++ b/ckan/templates/group/read_base.html @@ -26,12 +26,6 @@ {% link_for _('Activity Stream'), controller='group', action='activity', id=c.group_dict.name, icon='time' %} - - {% link_for _('Followers'), controller='group', action='followers', id=c.group_dict.name, icon='group' %} - - - {% link_for _('Administrators'), controller='group', action='admins', id=c.group_dict.name, icon='cog' %} - {% link_for _('About'), controller='group', action='about', id=c.group_dict.name, icon='info-sign' %} diff --git a/ckan/templates/organization/members.html b/ckan/templates/organization/members.html index fd0e34b6496..175af5e7566 100644 --- a/ckan/templates/organization/members.html +++ b/ckan/templates/organization/members.html @@ -11,9 +11,9 @@

{{ _('Members') }}

- {{ _('User') }} - {{ _('Role') }} - + {{ _('User') }} + {{ _('Role') }} + diff --git a/ckan/templates/organization/read_base.html b/ckan/templates/organization/read_base.html index d0c142f015c..23dc5c71ccd 100644 --- a/ckan/templates/organization/read_base.html +++ b/ckan/templates/organization/read_base.html @@ -33,7 +33,7 @@ {% endblock %} {% block secondary_content %} - {% snippet 'snippets/organization.html', organization=c.group_dict %} + {% snippet 'snippets/organization.html', organization=c.group_dict, show_nums=true %} {% if h.user_in_org_or_group(c.group_dict.id) %} {{ h.snippet('snippets/facet_list.html', title='Visibility', name='capacity', extras={'id':c.group_dict.id}) }} diff --git a/ckan/templates/package/read_base.html b/ckan/templates/package/read_base.html index 9a3d838ceb8..9f457eae772 100644 --- a/ckan/templates/package/read_base.html +++ b/ckan/templates/package/read_base.html @@ -43,7 +43,6 @@ {% snippet 'snippets/page_header.html', items=[ h.build_nav_icon('dataset_read', _('Dataset'), id=pkg.name), h.build_nav_icon('dataset_activity', _('Activity Stream'), id=pkg.name), - h.build_nav_icon('dataset_followers', _('Followers'), id=pkg.name), h.build_nav_icon('related_list', _('Related'), id=pkg.name), ] %} {% endblock %} @@ -56,6 +55,12 @@ {% block secondary_help_content %}{% endblock %} + {% block package_organization %} + {% if pkg.organization %} + {% snippet "snippets/organization.html", organization=pkg.organization, truncate=70, show_nums=false %} + {% endif %} + {% endblock %} + {% block package_groups %} {% for group in pkg.groups %} {% snippet "snippets/group.html", group=group, truncate=70 %} diff --git a/ckan/templates/package/resource_read.html b/ckan/templates/package/resource_read.html index 0f6f5c6d849..32dbcfc0a03 100644 --- a/ckan/templates/package/resource_read.html +++ b/ckan/templates/package/resource_read.html @@ -79,13 +79,13 @@

Additional Information

- - + + {% for key, value in h.format_resource_items(res.items()) %} - + {% endfor %}
FieldValueFieldValue
{{ key }}{{ value }}
{{ key }}{{ value }}
diff --git a/ckan/templates/package/snippets/additional_info.html b/ckan/templates/package/snippets/additional_info.html index 82e1eabd7d8..64821f08f59 100644 --- a/ckan/templates/package/snippets/additional_info.html +++ b/ckan/templates/package/snippets/additional_info.html @@ -3,52 +3,52 @@

{{ _('Additional Info') }}

- - + + {% if pkg_dict.url %} - + {% endif %} {% if pkg_dict.author_email %} - + {% elif pkg_dict.author %} - + {% endif %} {% if pkg_dict.maintainer_email %} - + {% elif pkg_dict.maintainer %} - + {% endif %} {% if pkg_dict.version %} - + {% endif %} {% if h.check_access('package_update',{'id':pkg_dict.id}) %} - + {% endif %} @@ -56,7 +56,7 @@

{{ _('Additional Info') }}

{% for extra in h.sorted_extras(pkg_dict.extras) %} {% set key, value = extra %} - + {% endfor %} diff --git a/ckan/templates/snippets/organization.html b/ckan/templates/snippets/organization.html index 8988a4c1338..8bbd4a9f418 100644 --- a/ckan/templates/snippets/organization.html +++ b/ckan/templates/snippets/organization.html @@ -26,18 +26,20 @@

{{ organization.title or organization.name }}

{% link_for _('read more'), controller='organization', action='about', id=organization.name %}

{% else %} -

{{ _('There is no description for this group') }}

+

{{ _('There is no description for this organization') }}

+ {% endif %} + {% if show_nums %} +
+
+
{{ _('Members') }}
+
{{ h.SI_number_span(organization.members|length) }}
+
+
+
{{ _('Datasets') }}
+
{{ h.SI_number_span(organization.package_count) }}
+
+
{% endif %} -
-
-
{{ _('Members') }}
-
{{ h.SI_number_span(organization.members|length) }}
-
-
-
{{ _('Datasets') }}
-
{{ h.SI_number_span(organization.packages|length) }}
-
-
{% endwith %} diff --git a/ckan/templates/snippets/related.html b/ckan/templates/snippets/related.html index 4a4395cc2e3..380a6276709 100644 --- a/ckan/templates/snippets/related.html +++ b/ckan/templates/snippets/related.html @@ -1,5 +1,5 @@
-

Related Add Related

+

Related

{% if item %} {% with url = h.url_for(controller='related', action='list', id=pkg_name) %} @@ -12,9 +12,11 @@

{{ item.title }}

{% endwith %} {% else %} -

No apps, ideas, news stories or images have been - related to this dataset yet, why not add one now?

-

{% link_for _('Add Item'), controller='related', action='new', id=pkg_name, icon='plus', class_='btn' %}

+

{% trans %}No apps, ideas, news stories or images have been + related to this dataset yet.{% endtrans %}

+ {% if h.check_access('related_create') %} +

{% link_for _('Add Item'), controller='related', action='new', id=pkg_name, icon='plus', class_='btn' %}

+ {% endif %} {% endif %}
diff --git a/ckan/templates/user/read.html b/ckan/templates/user/read.html index 9a272ecfa3e..c2afb1bca6c 100644 --- a/ckan/templates/user/read.html +++ b/ckan/templates/user/read.html @@ -11,7 +11,9 @@

{{ _('Datasets') }}

{% if c.is_myself %}

{{ _('You haven\'t created any datasets.') }} - {% link_for _('Create one now?'), controller='package', action='new' %}. + {% if h.check_access('package_create') %} + {% link_for _('Create one now?'), controller='package', action='new' %} + {% endif %}

{% else %}

@@ -20,4 +22,4 @@

{{ _('Datasets') }}

{% endif %} {% endif %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ckan/templates/user/read_base.html b/ckan/templates/user/read_base.html index cf287f33170..6baf84d3aa5 100644 --- a/ckan/templates/user/read_base.html +++ b/ckan/templates/user/read_base.html @@ -24,7 +24,6 @@ {% snippet 'snippets/page_header.html', items=[ h.build_nav_icon('user_datasets', _('Datasets'), id=user.name), h.build_nav_icon('user_activity_stream', _('Activity Stream'), id=user.name), - h.build_nav_icon('user_followers', _('Followers'), id=user.name), ] %} {% endblock %}
diff --git a/ckan/tests/__init__.py b/ckan/tests/__init__.py index 0d4be8b4e49..1a0aa934cc1 100644 --- a/ckan/tests/__init__.py +++ b/ckan/tests/__init__.py @@ -330,6 +330,10 @@ def is_migration_supported(): is_supported_db = not model.engine_is_sqlite() return is_supported_db +def is_datastore_supported(): + is_supported_db = model.engine_is_pg() + return is_supported_db + def search_related(test): def skip_test(*args): raise SkipTest("Search not supported") diff --git a/ckan/tests/functional/api/model/test_package.py b/ckan/tests/functional/api/model/test_package.py index e5a22bf0c7b..84563eb94ae 100644 --- a/ckan/tests/functional/api/model/test_package.py +++ b/ckan/tests/functional/api/model/test_package.py @@ -279,7 +279,6 @@ def test_register_post_indexerror(self): self.post_json(offset, data, status=500, extra_environ=self.admin_extra_environ) model.Session.remove() finally: - plugins.unload('synchronous_search') SolrSettings.init(original_settings) def test_register_post_tag_too_long(self): diff --git a/ckan/tests/functional/api/model/test_tag.py b/ckan/tests/functional/api/model/test_tag.py index 365d24df594..f9f0afdaa2a 100644 --- a/ckan/tests/functional/api/model/test_tag.py +++ b/ckan/tests/functional/api/model/test_tag.py @@ -4,6 +4,7 @@ from ckan import model from ckan.lib.create_test_data import CreateTestData +import ckan.lib.search as search from ckan.tests.functional.api.base import BaseModelApiTestCase from ckan.tests.functional.api.base import Api1TestCase as Version1TestCase @@ -14,6 +15,7 @@ class TagsTestCase(BaseModelApiTestCase): @classmethod def setup_class(cls): + search.clear() CreateTestData.create() cls.testsysadmin = model.User.by_name(u'testsysadmin') cls.comment = u'Comment umlaut: \xfc.' @@ -22,6 +24,7 @@ def setup_class(cls): @classmethod def teardown_class(cls): + search.clear() model.repo.rebuild_db() def test_register_get_ok(self): diff --git a/ckan/tests/functional/api/test_activity.py b/ckan/tests/functional/api/test_activity.py index 306d8bf3eb9..e5473df8d4c 100644 --- a/ckan/tests/functional/api/test_activity.py +++ b/ckan/tests/functional/api/test_activity.py @@ -17,7 +17,6 @@ from ckan.lib.helpers import json -##<<<<<<< HEAD ##def package_update(context, data_dict): ## # These tests call package_update directly which is really bad ## # setting api_version in context make things seem like the api key @@ -33,7 +32,6 @@ ## context['api_version'] = 3 ## context['ignore_auth'] = True ## return _package_create(context, data_dict) -##======= def package_show(app, data_dict, apikey=None): if apikey: extra_environ = {'Authorization': str(apikey)} @@ -2017,32 +2015,36 @@ def test_follow_dataset(self): # Find the new activity in the user's activity stream. user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) - assert len(user_new_activities) == 1, ("There should be 1 new " + assert len(user_new_activities) == 0, ("There should be 0 new " " activity in the user's activity stream, but found %i" % len(user_new_activities)) - activity = user_new_activities[0] + + # The rest of this test is commented out because 'follow dataset' + # activities are disabled, even they are reenabled then uncomment it. + + #activity = user_new_activities[0] # The same new activity should appear in the package's activity stream. - pkg_new_activities = after['package activity stream'] - for activity in user_new_activities: - assert activity in pkg_new_activities + #pkg_new_activities = after['package activity stream'] + #for activity in user_new_activities: + # assert activity in pkg_new_activities # Check that the new activity has the right attributes. - assert activity['object_id'] == self.warandpeace['id'], \ - str(activity['object_id']) - assert activity['user_id'] == user['id'], str(activity['user_id']) - assert activity['activity_type'] == 'follow dataset', \ - str(activity['activity_type']) - if 'id' not in activity: - assert False, "activity object should have an id value" + #assert activity['object_id'] == self.warandpeace['id'], \ + # str(activity['object_id']) + #assert activity['user_id'] == user['id'], str(activity['user_id']) + #assert activity['activity_type'] == 'follow dataset', \ + # str(activity['activity_type']) + #if 'id' not in activity: + # assert False, "activity object should have an id value" # TODO: Test for the _correct_ revision_id value. - if 'revision_id' not in activity: - assert False, "activity object should have a revision_id value" - timestamp = datetime_from_string(activity['timestamp']) - assert timestamp >= before['time'] and timestamp <= \ - after['time'], str(activity['timestamp']) + #if 'revision_id' not in activity: + # assert False, "activity object should have a revision_id value" + #timestamp = datetime_from_string(activity['timestamp']) + #assert timestamp >= before['time'] and timestamp <= \ + # after['time'], str(activity['timestamp']) - assert len(self.activity_details(activity)) == 0 + #assert len(self.activity_details(activity)) == 0 def test_follow_user(self): user = self.normal_user @@ -2063,27 +2065,32 @@ def test_follow_user(self): # Find the new activity in the user's activity stream. user_new_activities = (find_new_activities( before['user activity stream'], after['user activity stream'])) - assert len(user_new_activities) == 1, ("There should be 1 new " - " activity in the user's activity stream, but found %i" % + assert len(user_new_activities) == 0, ("There should be 0 new " + " activities in the user's activity stream, but found %i" % len(user_new_activities)) - activity = user_new_activities[0] + + # The rest of this test is commented out because follow_user activities + # are disabled, uncomment it if they're enabled again. + + #activity = user_new_activities[0] # Check that the new activity has the right attributes. - assert activity['object_id'] == self.sysadmin_user['id'], \ - str(activity['object_id']) - assert activity['user_id'] == user['id'], str(activity['user_id']) - assert activity['activity_type'] == 'follow user', \ - str(activity['activity_type']) - if 'id' not in activity: - assert False, "activity object should have an id value" + #assert activity['object_id'] == self.sysadmin_user['id'], \ + # str(activity['object_id']) + #assert activity['user_id'] == user['id'], str(activity['user_id']) + #assert activity['activity_type'] == 'follow user', \ + # str(activity['activity_type']) + #if 'id' not in activity: + # assert False, "activity object should have an id value" # TODO: Test for the _correct_ revision_id value. - if 'revision_id' not in activity: - assert False, "activity object should have a revision_id value" - timestamp = datetime_from_string(activity['timestamp']) - assert timestamp >= before['time'] and timestamp <= \ - after['time'], str(activity['timestamp']) - assert len(self.activity_details(activity)) == 0 + #if 'revision_id' not in activity: + # assert False, "activity object should have a revision_id value" + #timestamp = datetime_from_string(activity['timestamp']) + #assert timestamp >= before['time'] and timestamp <= \ + # after['time'], str(activity['timestamp']) + + #assert len(self.activity_details(activity)) == 0 def test_user_activity_list_by_name(self): '''user_activity_list should accept a user name as param.''' diff --git a/ckan/tests/functional/api/test_dashboard.py b/ckan/tests/functional/api/test_dashboard.py index adf9953ce9a..f44e163e293 100644 --- a/ckan/tests/functional/api/test_dashboard.py +++ b/ckan/tests/functional/api/test_dashboard.py @@ -172,23 +172,23 @@ def test_03_dashboard_activity_list_own_activities(self): '''Test that a user's own activities appear in her dashboard.''' activities = self.dashboard_activity_list(self.new_user) - # FIXME: There should actually be 6 activities here, but when you + # FIXME: There should actually be 3 activities here, but when you # follow something it's old activities (from before you followed it) # appear in your activity stream. So here we get more activities than # expected. - assert len(activities) == 8 + assert len(activities) == 5, len(activities) assert activities[0]['activity_type'] == 'changed package' - assert activities[1]['activity_type'] == 'follow group' - assert activities[2]['activity_type'] == 'follow user' - assert activities[3]['activity_type'] == 'follow dataset' - assert activities[4]['activity_type'] == 'new package' - assert activities[5]['activity_type'] == 'new user' + #assert activities[1]['activity_type'] == 'follow group' + #assert activities[2]['activity_type'] == 'follow user' + #assert activities[3]['activity_type'] == 'follow dataset' + assert activities[1]['activity_type'] == 'new package' + assert activities[2]['activity_type'] == 'new user' - # FIXME: Shouldn't need the [:6] here, it's because when you follow + # FIXME: Shouldn't need the [:3] here, it's because when you follow # something its old activities (from before you started following it) # appear in your dashboard. - for activity in activities[:6]: + for activity in activities[:3]: assert activity['user_id'] == self.new_user['id'] def test_03_own_activities_not_marked_as_new(self): diff --git a/ckan/tests/functional/api/test_follow.py b/ckan/tests/functional/api/test_follow.py index 1313c428845..79e6f5dab17 100644 --- a/ckan/tests/functional/api/test_follow.py +++ b/ckan/tests/functional/api/test_follow.py @@ -25,7 +25,6 @@ def datetime_from_string(s): ''' return datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%f') - def follow(func): '''Return a wrapper function for a follow_* function. @@ -33,18 +32,19 @@ def follow(func): calls, in addition to any tests carried out by the wrapped function. ''' - def wrapped_func(app, follower_id, apikey, object_id, object_arg): + def wrapped_func(app, follower_id, apikey, object_id, object_arg, + sysadmin_apikey): followee_count_before = ckan.tests.call_action_api(app, 'followee_count', id=follower_id) followees_before = ckan.tests.call_action_api(app, 'followee_list', - id=follower_id) + id=follower_id, apikey=sysadmin_apikey) - func(app, follower_id, apikey, object_id, object_arg) + func(app, follower_id, apikey, object_id, object_arg, sysadmin_apikey) followee_count_after = ckan.tests.call_action_api(app, 'followee_count', id=follower_id) followees_after = ckan.tests.call_action_api(app, 'followee_list', - id=follower_id) + id=follower_id, apikey=sysadmin_apikey) assert followee_count_after == followee_count_before + 1, ( "After a user follows an object, the user's `followee_count` " @@ -62,7 +62,8 @@ def wrapped_func(app, follower_id, apikey, object_id, object_arg): @follow -def follow_user(app, follower_id, apikey, object_id, object_arg): +def follow_user(app, follower_id, apikey, object_id, object_arg, + sysadmin_apikey): '''Test a user starting to follow another user via the API. :param follower_id: id of the user that will be following something. @@ -102,13 +103,13 @@ def follow_user(app, follower_id, apikey, object_id, object_arg): # Check that the follower appears in the object's list of followers. followers = ckan.tests.call_action_api(app, 'user_follower_list', - id=object_id) + id=object_id, apikey=sysadmin_apikey) assert len(followers) == follower_count_before + 1 assert len([follower for follower in followers if follower['id'] == follower_id]) == 1 # Check that the object appears in the follower's list of followees. followees = ckan.tests.call_action_api(app, 'user_followee_list', - id=follower_id) + apikey=sysadmin_apikey, id=follower_id) assert len(followees) == followee_count_before + 1 assert len([followee for followee in followees if followee['id'] == object_id]) == 1 @@ -124,7 +125,8 @@ def follow_user(app, follower_id, apikey, object_id, object_arg): @follow -def follow_dataset(app, follower_id, apikey, dataset_id, dataset_arg): +def follow_dataset(app, follower_id, apikey, dataset_id, dataset_arg, + sysadmin_apikey): '''Test a user starting to follow a dataset via the API. :param follower_id: id of the user. @@ -164,13 +166,13 @@ def follow_dataset(app, follower_id, apikey, dataset_id, dataset_arg): # Check that the follower appears in the dataset's list of followers. followers = ckan.tests.call_action_api(app, 'dataset_follower_list', - id=dataset_id) + id=dataset_id, apikey=sysadmin_apikey) assert len(followers) == follower_count_before + 1 assert len([follower for follower in followers if follower['id'] == follower_id]) == 1 # Check that the dataset appears in the follower's list of followees. followees = ckan.tests.call_action_api(app, 'dataset_followee_list', - id=follower_id) + apikey=sysadmin_apikey, id=follower_id) assert len(followees) == followee_count_before + 1 assert len([followee for followee in followees if followee['id'] == dataset_id]) == 1 @@ -186,7 +188,7 @@ def follow_dataset(app, follower_id, apikey, dataset_id, dataset_arg): @follow -def follow_group(app, user_id, apikey, group_id, group_arg): +def follow_group(app, user_id, apikey, group_id, group_arg, sysadmin_apikey): '''Test a user starting to follow a group via the API. :param user_id: id of the user @@ -226,14 +228,14 @@ def follow_group(app, user_id, apikey, group_id, group_arg): # Check that the user appears in the group's list of followers. followers = ckan.tests.call_action_api(app, 'group_follower_list', - id=group_id) + id=group_id, apikey=sysadmin_apikey) assert len(followers) == follower_count_before + 1 assert len([follower for follower in followers if follower['id'] == user_id]) == 1 # Check that the group appears in the user's list of followees. followees = ckan.tests.call_action_api(app, 'group_followee_list', - id=user_id) + apikey=sysadmin_apikey, id=user_id) assert len(followees) == followee_count_before + 1 assert len([followee for followee in followees if followee['id'] == group_id]) == 1 @@ -297,6 +299,126 @@ def setup_class(self): def teardown_class(self): ckan.model.repo.rebuild_db() + def test_00_visitor_cannot_get_user_follower_list(self): + ckan.tests.call_action_api(self.app, 'user_follower_list', + id=self.russianfan['id'], status=403) + + def test_00_user_cannot_get_user_follower_list(self): + ckan.tests.call_action_api(self.app, 'user_follower_list', + id=self.russianfan['id'], status=403, + apikey=self.annafan['apikey']) + + def test_00_sysadmin_can_get_user_follower_list(self): + ckan.tests.call_action_api(self.app, 'user_follower_list', + id=self.russianfan['id'], status=200, + apikey=self.testsysadmin['apikey']) + + def test_00_visitor_cannot_get_dataset_follower_list(self): + ckan.tests.call_action_api(self.app, 'dataset_follower_list', + id='warandpeace', status=403) + + def test_00_user_cannot_get_dataset_follower_list(self): + ckan.tests.call_action_api(self.app, 'dataset_follower_list', + id='warandpeace', status=403, apikey=self.annafan['apikey']) + + def test_00_sysadmin_can_get_dataset_follower_list(self): + ckan.tests.call_action_api(self.app, 'dataset_follower_list', + id='warandpeace', status=200, + apikey=self.testsysadmin['apikey']) + + def test_00_visitor_cannot_get_group_follower_list(self): + ckan.tests.call_action_api(self.app, 'group_follower_list', + id='roger', status=403) + + def test_00_user_cannot_get_group_follower_list(self): + ckan.tests.call_action_api(self.app, 'group_follower_list', + id='roger', status=403, apikey=self.annafan['apikey']) + + def test_00_sysadmin_can_get_group_follower_list(self): + ckan.tests.call_action_api(self.app, 'group_follower_list', + id='roger', status=200, apikey=self.testsysadmin['apikey']) + + def test_00_visitor_cannot_get_followee_list(self): + ckan.tests.call_action_api(self.app, 'followee_list', + id=self.russianfan['id'], status=403) + + def test_00_user_cannot_get_followee_list(self): + ckan.tests.call_action_api(self.app, 'followee_list', + id=self.russianfan['id'], status=403, + apikey=self.annafan['apikey']) + + def test_00_sysadmin_can_get_followee_list(self): + ckan.tests.call_action_api(self.app, 'followee_list', + id=self.russianfan['id'], status=200, + apikey=self.testsysadmin['apikey']) + + def test_00_visitor_cannot_get_user_followee_list(self): + '''A visitor cannot see what users a user is following.''' + ckan.tests.call_action_api(self.app, 'user_followee_list', + id=self.russianfan['id'], status=403) + + def test_00_user_cannot_get_user_followee_list(self): + '''A user cannot see what users another user is following.''' + ckan.tests.call_action_api(self.app, 'user_followee_list', + id=self.russianfan['id'], status=403, + apikey=self.annafan['apikey']) + + def test_00_sysadmin_can_get_user_followee_list(self): + '''A sysadmin can see what users another user is following.''' + ckan.tests.call_action_api(self.app, 'user_followee_list', + id=self.russianfan['id'], status=200, + apikey=self.testsysadmin['apikey']) + + def test_00_user_can_get_own_user_followee_list(self): + '''A user can see what users she herself is following.''' + ckan.tests.call_action_api(self.app, 'user_followee_list', + id=self.russianfan['id'], status=200, + apikey=self.russianfan['apikey']) + + def test_00_visitor_cannot_get_dataset_followee_list(self): + '''A visitor cannot see what datasets a user is following.''' + ckan.tests.call_action_api(self.app, 'dataset_followee_list', + id=self.russianfan['id'], status=403) + + def test_00_user_cannot_get_dataset_followee_list(self): + '''A user cannot see what datasets another user is following.''' + ckan.tests.call_action_api(self.app, 'dataset_followee_list', + id='russianfan', status=403, apikey=self.annafan['apikey']) + + def test_00_sysadmin_can_get_dataset_followee_list(self): + '''A sysadmin can see what datasets another user is following.''' + ckan.tests.call_action_api(self.app, 'dataset_followee_list', + id='russianfan', status=200, + apikey=self.testsysadmin['apikey']) + + def test_00_user_can_get_own_dataset_followee_list(self): + '''A user can see what datasets she herself is following.''' + ckan.tests.call_action_api(self.app, 'dataset_followee_list', + id=self.russianfan['id'], status=200, + apikey=self.russianfan['apikey']) + + def test_00_visitor_cannot_get_group_followee_list(self): + '''A visitor cannot see what groups a user is following.''' + ckan.tests.call_action_api(self.app, 'group_followee_list', + id='roger', status=403) + + def test_00_user_cannot_get_group_followee_list(self): + '''A user cannot see what groups another user is following.''' + ckan.tests.call_action_api(self.app, 'group_followee_list', + id='roger', status=403, apikey=self.annafan['apikey']) + + def test_00_sysadmin_can_get_group_followee_list(self): + '''A sysadmin can see what groups another user is following.''' + ckan.tests.call_action_api(self.app, 'group_followee_list', + id=self.annafan['id'], status=200, + apikey=self.testsysadmin['apikey']) + + def test_00_user_can_get_own_group_followee_list(self): + '''A user can see what groups she herself is following.''' + ckan.tests.call_action_api(self.app, 'group_followee_list', + id=self.russianfan['id'], status=200, + apikey=self.russianfan['apikey']) + def test_01_user_follow_user_bad_apikey(self): for apikey in ('bad api key', '', ' ', 'None', '3', '35.7', 'xxx'): error = ckan.tests.call_action_api(self.app, 'follow_user', @@ -357,27 +479,33 @@ def test_01_follow_missing_object_id(self): def test_02_user_follow_user_by_id(self): follow_user(self.app, self.annafan['id'], self.annafan['apikey'], - self.russianfan['id'], self.russianfan['id']) + self.russianfan['id'], self.russianfan['id'], + self.testsysadmin['apikey']) def test_02_user_follow_dataset_by_id(self): follow_dataset(self.app, self.annafan['id'], self.annafan['apikey'], - self.warandpeace['id'], self.warandpeace['id']) + self.warandpeace['id'], self.warandpeace['id'], + self.testsysadmin['apikey']) def test_02_user_follow_group_by_id(self): follow_group(self.app, self.annafan['id'], self.annafan['apikey'], - self.rogers_group['id'], self.rogers_group['id']) + self.rogers_group['id'], self.rogers_group['id'], + self.testsysadmin['apikey']) def test_02_user_follow_user_by_name(self): follow_user(self.app, self.annafan['id'], self.annafan['apikey'], - self.testsysadmin['id'], self.testsysadmin['name']) + self.testsysadmin['id'], self.testsysadmin['name'], + self.testsysadmin['apikey']) def test_02_user_follow_dataset_by_name(self): follow_dataset(self.app, self.joeadmin['id'], self.joeadmin['apikey'], - self.warandpeace['id'], self.warandpeace['name']) + self.warandpeace['id'], self.warandpeace['name'], + self.testsysadmin['apikey']) def test_02_user_follow_group_by_name(self): follow_group(self.app, self.joeadmin['id'], self.joeadmin['apikey'], - self.rogers_group['id'], self.rogers_group['name']) + self.rogers_group['id'], self.rogers_group['name'], + self.testsysadmin['apikey']) def test_03_user_follow_user_already_following(self): for object_id in (self.russianfan['id'], self.russianfan['name'], @@ -492,34 +620,38 @@ def test_04_follower_list_bad_id(self): 'group_follower_list'): for object_id in ('bad id', ' ', 3, 35.7, 'xxx', ''): error = ckan.tests.call_action_api(self.app, action, - status=409, id=object_id) + status=409, id=object_id, + apikey=self.testsysadmin['apikey']) assert error['id'] def test_04_follower_list_missing_id(self): for action in ('user_follower_list', 'dataset_follower_list', 'group_follower_list'): - error = ckan.tests.call_action_api(self.app, action, status=409) + error = ckan.tests.call_action_api(self.app, action, status=409, + apikey=self.testsysadmin['apikey']) assert error['id'] == ['Missing value'] def test_04_user_follower_list_no_followers(self): followers = ckan.tests.call_action_api(self.app, 'user_follower_list', - id=self.annafan['id']) + id=self.annafan['id'], apikey=self.testsysadmin['apikey']) assert followers == [] def test_04_dataset_follower_list_no_followers(self): followers = ckan.tests.call_action_api(self.app, - 'dataset_follower_list', id=self.annakarenina['id']) + 'dataset_follower_list', id=self.annakarenina['id'], + apikey=self.testsysadmin['apikey']) assert followers == [] def test_04_group_follower_list_no_followers(self): followers = ckan.tests.call_action_api(self.app, 'group_follower_list', - id=self.davids_group['id']) + id=self.davids_group['id'], apikey=self.testsysadmin['apikey']) assert followers == [] def _followee_list_bad_id(self, action): for object_id in ('bad id', ' ', 3, 35.7, 'xxx', ''): error = ckan.tests.call_action_api(self.app, action, - status=409, id=object_id) + status=409, id=object_id, + apikey=self.testsysadmin['apikey']) assert error['id'] def test_04_followee_list_bad_id(self): @@ -535,7 +667,8 @@ def test_04_group_followee_list_bad_id(self): self._followee_list_bad_id('group_followee_list') def _followee_list_missing_id(self, action): - error = ckan.tests.call_action_api(self.app, action, status=409) + error = ckan.tests.call_action_api(self.app, action, status=409, + apikey=self.testsysadmin['apikey']) assert error['id'] == ['Missing value'] def test_04_followee_list_missing_id(self): @@ -552,7 +685,7 @@ def test_04_group_followee_missing_bad_id(self): def _followee_list_not_following_anything(self, action): followees = ckan.tests.call_action_api(self.app, action, - id=self.russianfan['id']) + id=self.russianfan['id'], apikey=self.russianfan['apikey']) assert followees == [] def test_04_followee_list_not_following_anything(self): @@ -672,26 +805,34 @@ def setup_class(self): self.app = paste.fixture.TestApp(pylons.test.pylonsapp) follow_user(self.app, self.testsysadmin['id'], self.testsysadmin['apikey'], self.joeadmin['id'], - self.joeadmin['id']) + self.joeadmin['id'], self.testsysadmin['apikey']) follow_user(self.app, self.tester['id'], self.tester['apikey'], - self.joeadmin['id'], self.joeadmin['id']) + self.joeadmin['id'], self.joeadmin['id'], + self.testsysadmin['apikey']) follow_user(self.app, self.russianfan['id'], self.russianfan['apikey'], - self.joeadmin['id'], self.joeadmin['id']) + self.joeadmin['id'], self.joeadmin['id'], + self.testsysadmin['apikey']) follow_user(self.app, self.annafan['id'], self.annafan['apikey'], - self.joeadmin['id'], self.joeadmin['id']) + self.joeadmin['id'], self.joeadmin['id'], + self.testsysadmin['apikey']) follow_user(self.app, self.annafan['id'], self.annafan['apikey'], - self.tester['id'], self.tester['id']) + self.tester['id'], self.tester['id'], + self.testsysadmin['apikey']) follow_dataset(self.app, self.testsysadmin['id'], self.testsysadmin['apikey'], self.warandpeace['id'], - self.warandpeace['id']) + self.warandpeace['id'], self.testsysadmin['apikey']) follow_dataset(self.app, self.tester['id'], self.tester['apikey'], - self.warandpeace['id'], self.warandpeace['id']) + self.warandpeace['id'], self.warandpeace['id'], + self.testsysadmin['apikey']) follow_dataset(self.app, self.russianfan['id'], self.russianfan['apikey'], - self.warandpeace['id'], self.warandpeace['id']) + self.warandpeace['id'], self.warandpeace['id'], + self.testsysadmin['apikey']) follow_dataset(self.app, self.annafan['id'], self.annafan['apikey'], - self.warandpeace['id'], self.warandpeace['id']) + self.warandpeace['id'], self.warandpeace['id'], + self.testsysadmin['apikey']) follow_group(self.app, self.annafan['id'], self.annafan['apikey'], - self.davids_group['id'], self.davids_group['id']) + self.davids_group['id'], self.davids_group['id'], + self.testsysadmin['apikey']) @classmethod def teardown_class(self): @@ -800,7 +941,7 @@ def _unfollow_user(self, follower_id, apikey, object_id, object_arg): # Check that the user doesn't appear in the object's list of followers. followers = ckan.tests.call_action_api(self.app, 'user_follower_list', - id=object_id) + id=object_id, apikey=self.testsysadmin['apikey']) assert len([follower for follower in followers if follower['id'] == follower_id]) == 0 @@ -812,11 +953,11 @@ def _unfollow_user(self, follower_id, apikey, object_id, object_arg): # Check that the user doesn't appear in the subject's list of # followees. followees = ckan.tests.call_action_api(self.app, 'followee_list', - id=follower_id) + id=follower_id, apikey=apikey) assert len([followee for followee in followees if followee['dict']['id'] == object_id]) == 0 followees = ckan.tests.call_action_api(self.app, 'user_followee_list', - id=follower_id) + id=follower_id, apikey=apikey) assert len([followee for followee in followees if followee['id'] == object_id]) == 0 @@ -863,7 +1004,8 @@ def _unfollow_dataset(self, user_id, apikey, dataset_id, dataset_arg): # Check that the user doesn't appear in the dataset's list of # followers. followers = ckan.tests.call_action_api(self.app, - 'dataset_follower_list', id=dataset_id) + 'dataset_follower_list', id=dataset_id, + apikey=self.testsysadmin['apikey']) assert len([follower for follower in followers if follower['id'] == user_id]) == 0 @@ -875,11 +1017,11 @@ def _unfollow_dataset(self, user_id, apikey, dataset_id, dataset_arg): # Check that the dataset doesn't appear in the user's list of # followees. followees = ckan.tests.call_action_api(self.app, 'followee_list', - id=user_id) + id=user_id, apikey=apikey) assert len([followee for followee in followees if followee['dict']['id'] == dataset_id]) == 0 followees = ckan.tests.call_action_api(self.app, - 'dataset_followee_list', id=user_id) + 'dataset_followee_list', id=user_id, apikey=apikey) assert len([followee for followee in followees if followee['id'] == dataset_id]) == 0 @@ -926,7 +1068,7 @@ def _unfollow_group(self, user_id, apikey, group_id, group_arg): # Check that the user doesn't appear in the group's list of # followers. followers = ckan.tests.call_action_api(self.app, 'group_follower_list', - id=group_id) + id=group_id, apikey=self.testsysadmin['apikey']) assert len([follower for follower in followers if follower['id'] == user_id]) == 0 @@ -938,11 +1080,12 @@ def _unfollow_group(self, user_id, apikey, group_id, group_arg): # Check that the group doesn't appear in the user's list of # followees. followees = ckan.tests.call_action_api(self.app, 'followee_list', - id=user_id) + id=user_id, apikey=apikey) assert len([followee for followee in followees if followee['dict']['id'] == group_id]) == 0 followees = ckan.tests.call_action_api(self.app, - 'group_followee_list', id=user_id) + 'group_followee_list', id=user_id, + apikey=self.testsysadmin['apikey']) assert len([followee for followee in followees if followee['id'] == group_id]) == 0 @@ -1012,31 +1155,38 @@ def setup_class(self): self.app = paste.fixture.TestApp(pylons.test.pylonsapp) follow_user(self.app, self.joeadmin['id'], self.joeadmin['apikey'], - self.testsysadmin['id'], self.testsysadmin['id']) + self.testsysadmin['id'], self.testsysadmin['id'], + self.testsysadmin['apikey']) follow_user(self.app, self.annafan['id'], self.annafan['apikey'], - self.testsysadmin['id'], self.testsysadmin['id']) + self.testsysadmin['id'], self.testsysadmin['id'], + self.testsysadmin['apikey']) follow_user(self.app, self.russianfan['id'], self.russianfan['apikey'], - self.testsysadmin['id'], self.testsysadmin['id']) + self.testsysadmin['id'], self.testsysadmin['id'], + self.testsysadmin['apikey']) follow_dataset(self.app, self.joeadmin['id'], self.joeadmin['apikey'], - self.annakarenina['id'], self.annakarenina['id']) + self.annakarenina['id'], self.annakarenina['id'], + self.testsysadmin['apikey']) follow_dataset(self.app, self.annafan['id'], self.annafan['apikey'], - self.annakarenina['id'], self.annakarenina['id']) + self.annakarenina['id'], self.annakarenina['id'], + self.testsysadmin['apikey']) follow_dataset(self.app, self.russianfan['id'], self.russianfan['apikey'], - self.annakarenina['id'], self.annakarenina['id']) + self.annakarenina['id'], self.annakarenina['id'], + self.testsysadmin['apikey']) follow_user(self.app, self.tester['id'], self.tester['apikey'], - self.joeadmin['id'], self.joeadmin['id']) + self.joeadmin['id'], self.joeadmin['id'], + self.testsysadmin['apikey']) follow_dataset(self.app, self.testsysadmin['id'], self.testsysadmin['apikey'], self.warandpeace['id'], - self.warandpeace['id']) + self.warandpeace['id'], self.testsysadmin['apikey']) follow_group(self.app, self.testsysadmin['id'], self.testsysadmin['apikey'], self.davids_group['id'], - self.davids_group['id']) + self.davids_group['id'], self.testsysadmin['apikey']) session = ckan.model.Session() session.delete(ckan.model.User.get('joeadmin')) @@ -1060,24 +1210,24 @@ def test_01_on_delete_cascade_api(self): ''' # It should no longer be possible to get joeadmin's follower list. error = ckan.tests.call_action_api(self.app, 'user_follower_list', - status=409, id='joeadmin') + status=409, id='joeadmin', apikey=self.testsysadmin['apikey']) assert 'id' in error # It should no longer be possible to get joeadmin's followee lists. for action in ('followee_list', 'user_followee_list', 'dataset_followee_list', 'group_followee_list'): error = ckan.tests.call_action_api(self.app, action, status=409, - id='joeadmin') + id='joeadmin', apikey=self.testsysadmin['apikey']) assert 'id' in error # It should no longer be possible to get warandpeace's follower list. error = ckan.tests.call_action_api(self.app, 'dataset_follower_list', - status=409, id='warandpeace') + status=409, id='warandpeace', apikey=self.testsysadmin['apikey']) assert 'id' in error # It should no longer be possible to get david's follower list. error = ckan.tests.call_action_api(self.app, 'group_follower_list', - status=409, id='david') + status=409, id='david', apikey=self.testsysadmin['apikey']) assert 'id' in error # It should no longer be possible to get joeadmin's follower count. @@ -1152,13 +1302,14 @@ def test_01_on_delete_cascade_api(self): # Users who joeadmin was following should no longer have him in their # follower list. followers = ckan.tests.call_action_api(self.app, 'user_follower_list', - id=self.testsysadmin['id']) + id=self.testsysadmin['id'], apikey=self.testsysadmin['apikey']) assert 'joeadmin' not in [follower['name'] for follower in followers] # Datasets who joeadmin was following should no longer have him in # their follower list. followers = ckan.tests.call_action_api(self.app, - 'dataset_follower_list', id=self.annakarenina['id']) + 'dataset_follower_list', id=self.annakarenina['id'], + apikey=self.testsysadmin['apikey']) assert 'joeadmin' not in [follower['name'] for follower in followers] def test_02_on_delete_cascade_db(self): diff --git a/ckan/tests/functional/test_activity.py b/ckan/tests/functional/test_activity.py index 7e4f0d89a3d..5634171c12d 100644 --- a/ckan/tests/functional/test_activity.py +++ b/ckan/tests/functional/test_activity.py @@ -120,14 +120,14 @@ def test_user_activity(self): result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s started following %s' % (user['fullname'], - package['title']) in stripped, stripped + package['title']) not in stripped, stripped # Follow another user. follow_user(context, {'id': 'joeadmin'}) result = self.app.get(offset, status=200) stripped = self.strip_tags(result) assert '%s started following %s' % (user['fullname'], - 'joeadmin') in stripped, stripped + 'joeadmin') not in stripped, stripped # Create a new group. group = { diff --git a/ckan/tests/functional/test_follow.py b/ckan/tests/functional/test_follow.py index 268fb823706..acb97deb0ac 100644 --- a/ckan/tests/functional/test_follow.py +++ b/ckan/tests/functional/test_follow.py @@ -68,24 +68,12 @@ def test_dataset_read_not_logged_in(self): assert 'id="dataset_follow_button"' not in result def test_dataset_followers_not_logged_in(self): + '''Not-logged-in users cannot see /dataset/followers/ pages.''' offset = url_for(controller='package', action='followers', id='warandpeace') result = self.app.get(offset) - assert 'href="/dataset/followers/warandpeace"' in result - assert 'Followers (0)' in result - assert 'id="dataset_follow_button"' not in result - assert '
  • ' not in result - - offset = url_for(controller='package', action='followers', - id='annakarenina') - result = self.app.get(offset) - assert 'href="/dataset/followers/annakarenina"' in result - assert 'Followers (3)' in result - assert 'id="dataset_follow_button"' not in result - assert str(result).count('
  • ') == 3 - assert self.joeadmin.display_name in result - assert self.annafan.display_name in result - assert self.russianfan.display_name in result + assert result.status == 302 + assert '/user/login' in result.header_dict['location'] def test_user_read_not_logged_in(self): offset = url_for(controller='user', action='read', @@ -106,20 +94,8 @@ def test_user_followers_not_logged_in(self): offset = url_for(controller='user', action='followers', id='joeadmin') result = self.app.get(offset) - assert 'href="/user/followers/joeadmin"' in result - assert 'Followers (0)' in result - assert '
  • ' not in result - assert 'id="user_follow_button"' not in result - - offset = url_for(controller='user', action='followers', - id='annafan') - result = self.app.get(offset) - assert 'href="/user/followers/annafan"' in result - assert 'Followers (2)' in result - assert 'id="user_follow_button"' not in result - assert str(result).count('
  • ') == 2 - assert self.tester.display_name in result - assert self.russianfan.display_name in result + assert result.status == 302 + assert '/user/login' in result.header_dict['location'] def test_own_user_read_logged_in(self): offset = url_for(controller='user', action='read', @@ -141,28 +117,17 @@ def test_own_user_read_logged_in(self): def test_own_user_followers_logged_in(self): offset = url_for(controller='user', action='followers', id='joeadmin') - extra_environ = {'Authorization': str(self.joeadmin.apikey)} + extra_environ = {'Authorization': str(self.testsysadmin.apikey)} result = self.app.get(offset, extra_environ=extra_environ) assert 'href="/user/followers/joeadmin"' in result - assert 'My Followers (0)' in result - assert 'id="user_follow_button"' not in result + assert 'Followers (0)' in result + assert 'id="user_follow_button"' in result assert '
  • ' not in result - offset = url_for(controller='user', action='followers', - id='annafan') - extra_environ = {'Authorization': str(self.annafan.apikey)} - result = self.app.get(offset, extra_environ=extra_environ) - assert 'href="/user/followers/annafan"' in result - assert 'My Followers (2)' in result - assert 'id="user_follow_button"' not in result - assert str(result).count('
  • ') == 2 - assert self.tester.display_name in result - assert self.russianfan.display_name in result - def test_dataset_read_logged_in(self): offset = url_for(controller='package', action='read', id='warandpeace') - extra_environ = {'Authorization': str(self.tester.apikey)} + extra_environ = {'Authorization': str(self.testsysadmin.apikey)} result = self.app.get(offset, extra_environ=extra_environ) assert 'href="/dataset/followers/warandpeace"' in result assert 'Followers (0)' in result @@ -179,16 +144,16 @@ def test_dataset_read_logged_in(self): def test_dataset_follow_logged_in(self): offset = url_for(controller='package', action='followers', id='warandpeace') - extra_environ = {'Authorization': str(self.tester.apikey)} + extra_environ = {'Authorization': str(self.testsysadmin.apikey)} result = self.app.get(offset, extra_environ=extra_environ) - assert 'href="/dataset/followers/warandpeace"' in result + assert 'id="dataset_follow_button"' in result assert 'Followers (0)' in result assert 'id="dataset_follow_button"' in result assert '
  • ' not in result offset = url_for(controller='package', action='followers', id='annakarenina') - extra_environ = {'Authorization': str(self.tester.apikey)} + extra_environ = {'Authorization': str(self.testsysadmin.apikey)} result = self.app.get(offset, extra_environ=extra_environ) assert 'href="/dataset/followers/annakarenina"' in result assert 'Followers (3)' in result @@ -202,7 +167,7 @@ def test_dataset_follow_logged_in(self): # button. offset = url_for(controller='package', action='followers', id='annakarenina') - extra_environ = {'Authorization': str(self.joeadmin.apikey)} + extra_environ = {'Authorization': str(self.testsysadmin.apikey)} result = self.app.get(offset, extra_environ=extra_environ) assert 'Unfollow' in result @@ -226,28 +191,9 @@ def test_user_read_logged_in(self): def test_user_follow_logged_in(self): offset = url_for(controller='user', action='followers', id='joeadmin') - extra_environ = {'Authorization': str(self.tester.apikey)} + extra_environ = {'Authorization': str(self.testsysadmin.apikey)} result = self.app.get(offset, extra_environ=extra_environ) assert 'href="/user/followers/joeadmin"' in result assert 'Followers (0)' in result assert '
  • ' not in result assert 'id="user_follow_button"' in result - - offset = url_for(controller='user', action='followers', - id='annafan') - extra_environ = {'Authorization': str(self.tester.apikey)} - result = self.app.get(offset, extra_environ=extra_environ) - assert 'href="/user/followers/annafan"' in result - assert 'Followers (2)' in result - assert 'id="user_follow_button"' in result - assert str(result).count('
  • ') == 2 - assert self.tester.display_name in result - assert self.russianfan.display_name in result - - # russianfan is following annafan so he should see an Unfollow - # button. - offset = url_for(controller='user', action='followers', - id='annafan') - extra_environ = {'Authorization': str(self.russianfan.apikey)} - result = self.app.get(offset, extra_environ=extra_environ) - assert 'Unfollow' in result diff --git a/ckan/tests/functional/test_group.py b/ckan/tests/functional/test_group.py index cbe1fb52f13..c45090a6a7d 100644 --- a/ckan/tests/functional/test_group.py +++ b/ckan/tests/functional/test_group.py @@ -8,6 +8,7 @@ import ckan.model as model from ckan.lib.create_test_data import CreateTestData from ckan.logic import check_access, NotAuthorized, get_action +import ckan.lib.search as search from pylons import config @@ -51,6 +52,7 @@ class TestGroup(FunctionalTestCase): @classmethod def setup_class(self): + search.clear() model.Session.remove() CreateTestData.create() diff --git a/ckan/tests/lib/test_dictization.py b/ckan/tests/lib/test_dictization.py index c78c9365dc0..61828f0a27d 100644 --- a/ckan/tests/lib/test_dictization.py +++ b/ckan/tests/lib/test_dictization.py @@ -1,6 +1,7 @@ from ckan.tests import assert_equal, assert_not_in, assert_in from pprint import pprint, pformat from difflib import unified_diff +import ckan.lib.search as search from ckan.lib.create_test_data import CreateTestData from ckan import model @@ -32,6 +33,7 @@ class TestBasicDictize: def setup_class(cls): # clean the db so we can run these tests on their own model.repo.rebuild_db() + search.clear() CreateTestData.create() cls.package_expected = { @@ -928,6 +930,7 @@ def test_16_group_dictized(self): 'name': u'help', 'display_name': u'help', 'image_url': u'', + 'package_count': 2, 'is_organization': False, 'packages': [{'author': None, 'author_email': None, @@ -969,10 +972,12 @@ def test_16_group_dictized(self): result['packages'] = sorted(result['packages'], key=lambda x: x['name']) assert_equal(sorted(result.keys()), sorted(expected.keys())) + for key in result: - if key == 'is_organization': + if key in ('is_organization', 'package_count'): continue assert_equal(sorted(result[key]), sorted(expected[key])) + assert_equal(result['package_count'], expected['package_count']) def test_17_group_apis_to_dict(self): diff --git a/ckan/tests/logic/test_action.py b/ckan/tests/logic/test_action.py index 6f418edfce3..1e115b24374 100644 --- a/ckan/tests/logic/test_action.py +++ b/ckan/tests/logic/test_action.py @@ -16,6 +16,7 @@ from ckan.logic import get_action, NotAuthorized from ckan.logic.action import get_domain_object from ckan.tests import TestRoles +import ckan.lib.search as search from ckan import plugins from ckan.plugins import SingletonPlugin, implements, IPackageController @@ -28,6 +29,7 @@ class TestAction(WsgiAppCase): @classmethod def setup_class(cls): + search.clear() CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') diff --git a/ckan/tests/logic/test_tag.py b/ckan/tests/logic/test_tag.py index 1f710ab6563..82d0cdd1e1e 100644 --- a/ckan/tests/logic/test_tag.py +++ b/ckan/tests/logic/test_tag.py @@ -1,6 +1,7 @@ import json from pprint import pprint from nose.tools import assert_equal, assert_raises +import ckan.lib.search as search import ckan.model as model from ckan.lib.create_test_data import CreateTestData @@ -10,6 +11,7 @@ class TestAction(WsgiAppCase): @classmethod def setup_class(cls): + search.clear() CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') cls.normal_user = model.User.get('annafan') diff --git a/ckanext/datastore/plugin.py b/ckanext/datastore/plugin.py index 98f8cd92af1..cdb547a2211 100644 --- a/ckanext/datastore/plugin.py +++ b/ckanext/datastore/plugin.py @@ -1,11 +1,13 @@ import logging import pylons from sqlalchemy.exc import ProgrammingError + import ckan.plugins as p import ckanext.datastore.logic.action as action import ckanext.datastore.logic.auth as auth import ckanext.datastore.db as db import ckan.logic as logic +import ckan.model as model log = logging.getLogger(__name__) _get_or_bust = logic.get_or_bust @@ -52,21 +54,24 @@ def configure(self, config): else: self.read_url = self.config['ckan.datastore.read_url'] - if not self._is_read_only_database(): - # Make sure that the right permissions are set - # so that no harmful queries can be made - if not ('debug' in config and config['debug']): - self._check_separate_db() - if self.legacy_mode: - log.warn('Legacy mode active. The sql search will not be available.') - else: - self._check_read_permissions() + if model.engine_is_pg(): + if not self._is_read_only_database(): + # Make sure that the right permissions are set + # so that no harmful queries can be made + if not ('debug' in config and config['debug']): + self._check_separate_db() + if self.legacy_mode: + log.warn('Legacy mode active. The sql search will not be available.') + else: + self._check_read_permissions() - self._create_alias_table() + self._create_alias_table() + else: + log.warn("We detected that CKAN is running on a read only database. " + "Permission checks and the creation of _table_metadata are skipped.") else: - log.warn("We detected that CKAN is running on a read only database. " - "Permission checks and _table_metadata creation are skipped." - "Make sure that replication is properly set-up.") + log.warn("We detected that you do not use a PostgreSQL database. " + "The DataStore will NOT work and datastore tests will be skipped.") ## Do light wrapping around action function to add datastore_active ## to resource dict. Not using IAction extension as this prevents other plugins diff --git a/ckanext/datastore/tests/test_create.py b/ckanext/datastore/tests/test_create.py index 4fd7da6b48c..d4afaf01fd4 100644 --- a/ckanext/datastore/tests/test_create.py +++ b/ckanext/datastore/tests/test_create.py @@ -1,4 +1,5 @@ import json +import nose import sqlalchemy.orm as orm @@ -18,6 +19,8 @@ class TestDatastoreCreate(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') diff --git a/ckanext/datastore/tests/test_delete.py b/ckanext/datastore/tests/test_delete.py index 41a43431d5e..01333142585 100644 --- a/ckanext/datastore/tests/test_delete.py +++ b/ckanext/datastore/tests/test_delete.py @@ -1,4 +1,5 @@ import json +import nose import sqlalchemy import sqlalchemy.orm as orm @@ -19,6 +20,8 @@ class TestDatastoreDelete(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') diff --git a/ckanext/datastore/tests/test_search.py b/ckanext/datastore/tests/test_search.py index 36fe8556a31..44e6a4f5ac8 100644 --- a/ckanext/datastore/tests/test_search.py +++ b/ckanext/datastore/tests/test_search.py @@ -1,4 +1,5 @@ import json +import nose import pprint import sqlalchemy.orm as orm @@ -18,6 +19,8 @@ class TestDatastoreSearch(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') @@ -331,6 +334,8 @@ def test_search_table_metadata(self): class TestDatastoreFullTextSearch(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') @@ -397,6 +402,8 @@ class TestDatastoreSQL(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') diff --git a/ckanext/datastore/tests/test_upsert.py b/ckanext/datastore/tests/test_upsert.py index fa8016a6d3c..de32c376f4c 100644 --- a/ckanext/datastore/tests/test_upsert.py +++ b/ckanext/datastore/tests/test_upsert.py @@ -1,4 +1,5 @@ import json +import nose import datetime import sqlalchemy.orm as orm @@ -19,6 +20,8 @@ class TestDatastoreUpsert(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') @@ -243,6 +246,8 @@ class TestDatastoreInsert(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') @@ -344,6 +349,8 @@ class TestDatastoreUpdate(tests.WsgiAppCase): @classmethod def setup_class(cls): + if not tests.is_datastore_supported(): + raise nose.SkipTest("Datastore not supported") p.load('datastore') ctd.CreateTestData.create() cls.sysadmin_user = model.User.get('testsysadmin') diff --git a/ckanext/reclinepreview/theme/public/preview_recline.js b/ckanext/reclinepreview/theme/public/preview_recline.js index 91c5063e816..bf56d89b66c 100644 --- a/ckanext/reclinepreview/theme/public/preview_recline.js +++ b/ckanext/reclinepreview/theme/public/preview_recline.js @@ -39,6 +39,8 @@ this.ckan.module('reclinepreview', function (jQuery, _) { .html(msg); } + recline.Backend.DataProxy.timeout = 10000; + // 2 situations // a) something was posted to the datastore - need to check for this // b) csv or xls (but not datastore) diff --git a/doc/install-from-source.rst b/doc/install-from-source.rst index 032d777fa1a..2475503c1b8 100644 --- a/doc/install-from-source.rst +++ b/doc/install-from-source.rst @@ -7,13 +7,13 @@ This section describes how to install CKAN from source. Although CKAN from source works with Ubuntu 10.04, with other versions of Ubuntu (e.g. 12.04) and with other operating systems (e.g. RedHat, Fedora, CentOS, OS X). If you install CKAN from source on your own operating system, please share your -experiences on our wiki: http://wiki.ckan.org/Install +experiences on our `How to Install CKAN `_ +wiki page. From source is also the right installation method for developers who want to work on CKAN. -If you run into problems, see :doc:`common-error-messages` or contact `the -ckan-dev mailing list `_. +If you run into problems, see :doc:`common-error-messages`. 1. Install the required packages ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -25,7 +25,8 @@ required packages with this command:: If you're not using a Debian-based operating system, find the best way to install the following packages on your operating system (see -http://wiki.ckan.org/Install for help): +our `How to Install CKAN `_ +wiki page for help): ===================== =============================================== Package Description
    • {{ _('Field') }}{{ _('Value') }}{{ _('Field') }}{{ _('Value') }}
    {{ _('Source') }}{{ _('Source') }} {{ h.link_to(pkg_dict.url, pkg_dict.url, rel='foaf:homepage', target='_blank') }}
    {{ _("Author") }}{{ _("Author") }} {{ h.mail_to(email_address=pkg_dict.author_email, name=pkg_dict.author) }}
    {{ _("Author") }}{{ _("Author") }} {{ pkg_dict.author }}
    {{ _('Maintainer') }}{{ _('Maintainer') }} {{ h.mail_to(email_address=pkg_dict.maintainer_email, name=pkg_dict.maintainer) }}
    {{ _('Maintainer') }}{{ _('Maintainer') }} {{ pkg_dict.maintainer }}
    {{ _("Version") }}{{ _("Version") }} {{ pkg_dict.version }}
    {{ _("State") }}{{ _("State") }} {{ pkg_dict.state }}
    {{ _(key) }}{{ _(key) }} {{ value }}