From 4aa6b81742a712068e03082429ad60c4508cc4bf Mon Sep 17 00:00:00 2001 From: David Read Date: Fri, 12 Oct 2018 17:44:20 +0100 Subject: [PATCH] [#4484] manually backported to ckan 2.7.x --- ckan/controllers/api.py | 8 + ckan/lib/dictization/model_dictize.py | 5 +- ckan/lib/navl/validators.py | 24 +- ckan/lib/search/query.py | 4 +- ckan/logic/action/get.py | 109 ++++--- ckan/logic/schema.py | 17 +- ckan/model/group.py | 4 +- .../legacy/functional/api/test_dashboard.py | 2 + ckan/tests/legacy/functional/test_activity.py | 3 +- .../lib/dictization/test_model_dictize.py | 11 + ckan/tests/logic/action/test_get.py | 291 ++++++++++++++++++ doc/maintaining/configuration.rst | 63 +++- test-core.ini | 2 - 13 files changed, 495 insertions(+), 48 deletions(-) diff --git a/ckan/controllers/api.py b/ckan/controllers/api.py index b02b339d49e..d72d5724e33 100644 --- a/ckan/controllers/api.py +++ b/ckan/controllers/api.py @@ -581,6 +581,14 @@ def search(self, ver=None, register=None): # the search if 'callback' in params: del params['callback'] + + # I needed to add limit here, because it bypasses the logic + # function and therefore the validator (which now does the + # limiting when calling package_search) + params['rows'] = min( + int(params.get('rows', 10)), + int(config.get('ckan.search.rows_max', 1000))) + results = query.run(params) return self._finish_ok(results) except search.SearchError, e: diff --git a/ckan/lib/dictization/model_dictize.py b/ckan/lib/dictization/model_dictize.py index f39c9c3057a..80776f8a79c 100644 --- a/ckan/lib/dictization/model_dictize.py +++ b/ckan/lib/dictization/model_dictize.py @@ -389,11 +389,12 @@ def get_packages_for_this_group(group_, just_the_count=False): q['include_private'] = True if not just_the_count: - # Is there a packages limit in the context? + # package_search limits 'rows' anyway, so this is only if you + # want even fewer try: packages_limit = context['limits']['packages'] except KeyError: - q['rows'] = 1000 # Only the first 1000 datasets are returned + del q['rows'] # leave it to package_search to limit it else: q['rows'] = packages_limit diff --git a/ckan/lib/navl/validators.py b/ckan/lib/navl/validators.py index cfcb1a2d5d2..d60cfcfd719 100644 --- a/ckan/lib/navl/validators.py +++ b/ckan/lib/navl/validators.py @@ -2,7 +2,7 @@ import ckan.lib.navl.dictization_functions as df -from ckan.common import _ +from ckan.common import _, config missing = df.missing StopOnError = df.StopOnError @@ -81,6 +81,15 @@ def callable(key, data, errors, context): return callable +def configured_default(config_name, default_value_if_not_configured): + '''When key is missing or value is an empty string or None, replace it with + a default value from config, or if that isn't set from the + default_value_if_not_configured.''' + default_value = config.get(config_name) + if default_value is None: + default_value = default_value_if_not_configured + return default(default_value) + def ignore_missing(key, data, errors, context): '''If the key is missing from the data, ignore the rest of the key's schema. @@ -123,3 +132,16 @@ def unicode_only(value): if not isinstance(value, unicode): raise Invalid(_('Must be a Unicode string value')) return value + +def limit_to_configured_maximum(config_option, default_limit): + ''' + If the value is over a limit, it changes it to the limit. The limit is + defined by a configuration option, or if that is not set, a given int + default_limit. + ''' + def callable(key, data, errors, context): + value = data.get(key) + limit = int(config.get(config_option, default_limit)) + if value > limit: + data[key] = limit + return callable diff --git a/ckan/lib/search/query.py b/ckan/lib/search/query.py index 527670371bb..818f3bee0b4 100644 --- a/ckan/lib/search/query.py +++ b/ckan/lib/search/query.py @@ -312,7 +312,9 @@ def run(self, query, permission_labels=None, **kwargs): query['q'] = "*:*" # number of results - rows_to_return = min(1000, int(query.get('rows', 10))) + rows_to_return = int(query.get('rows', 10)) + # query['rows'] should be a defaulted int, due to schema, but make + # certain, for legacy tests if rows_to_return > 0: # #1683 Work around problem of last result being out of order # in SOLR 1.4 diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index d7b92a29e9a..1847e0b446e 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -341,6 +341,15 @@ def _group_or_org_list(context, data_dict, is_org=False): all_fields = asbool(data_dict.get('all_fields', None)) + if all_fields: + # all_fields is really computationally expensive, so need a tight limit + max_limit = config.get( + 'ckan.group_and_organization_list_all_fields_max', 25) + else: + max_limit = config.get('ckan.group_and_organization_list_max', 1000) + if limit is None or limit > max_limit: + limit = max_limit + # order_by deprecated in ckan 1.8 # if it is supplied and sort isn't use order_by and raise a warning order_by = data_dict.get('order_by', '') @@ -437,6 +446,11 @@ def group_list(context, data_dict): "name asc" string of field name and sort-order. The allowed fields are 'name', 'package_count' and 'title' :type sort: string + :param limit: the maximum number of groups returned (optional) + Default: ``1000`` when all_fields=false unless set in site's + configuration ``ckan.group_and_organization_list_max`` + Default: ``25`` when all_fields=true unless set in site's + configuration ``ckan.group_and_organization_list_all_fields_max`` :param limit: if given, the list of groups will be broken into pages of at most ``limit`` groups per page and only one page will be returned at a time (optional) @@ -486,6 +500,11 @@ def organization_list(context, data_dict): "name asc" string of field name and sort-order. The allowed fields are 'name', 'package_count' and 'title' :type sort: string + :param limit: the maximum number of organizations returned (optional) + Default: ``1000`` when all_fields=false unless set in site's + configuration ``ckan.group_and_organization_list_max`` + Default: ``25`` when all_fields=true unless set in site's + configuration ``ckan.group_and_organization_list_all_fields_max`` :param limit: if given, the list of organizations will be broken into pages of at most ``limit`` organizations per page and only one page will be returned at a time (optional) @@ -1739,8 +1758,9 @@ def package_search(context, data_dict): documentation, this is a comma-separated string of field names and sort-orderings. :type sort: string - :param rows: the number of matching rows to return. There is a hard limit - of 1000 datasets per query. + :param rows: the maximum number of matching rows (datasets) to return. + (optional, default: ``10``, upper limit: ``1000`` unless set in + site's configuration ``ckan.search.rows_max``) :type rows: int :param start: the offset in the complete result for where the set of returned datasets should begin. @@ -2500,8 +2520,9 @@ def user_activity_list(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: list of dictionaries @@ -2519,8 +2540,7 @@ def user_activity_list(context, data_dict): raise logic.NotFound offset = data_dict.get('offset', 0) - limit = int( - data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) + limit = data_dict['limit'] # defaulted, limited & made an int by schema _activity_objects = model.activity.user_activity_list(user.id, limit=limit, offset=offset) @@ -2542,8 +2562,9 @@ def package_activity_list(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: list of dictionaries @@ -2561,8 +2582,7 @@ def package_activity_list(context, data_dict): raise logic.NotFound offset = int(data_dict.get('offset', 0)) - limit = int( - data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) + limit = data_dict['limit'] # defaulted, limited & made an int by schema _activity_objects = model.activity.package_activity_list(package.id, limit=limit, offset=offset) @@ -2584,8 +2604,9 @@ def group_activity_list(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: list of dictionaries @@ -2598,8 +2619,7 @@ def group_activity_list(context, data_dict): model = context['model'] group_id = data_dict.get('id') offset = data_dict.get('offset', 0) - limit = int( - data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) + limit = data_dict['limit'] # defaulted, limited & made an int by schema # Convert group_id (could be id or name) into id. group_show = logic.get_action('group_show') @@ -2619,6 +2639,14 @@ def organization_activity_list(context, data_dict): :param id: the id or name of the organization :type id: string + :param offset: where to start getting activity items from + (optional, default: ``0``) + :type offset: int + :param limit: the maximum number of activities to return + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) + :type limit: int :rtype: list of dictionaries @@ -2630,8 +2658,7 @@ def organization_activity_list(context, data_dict): model = context['model'] org_id = data_dict.get('id') offset = data_dict.get('offset', 0) - limit = int( - data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) + limit = data_dict['limit'] # defaulted, limited & made an int by schema # Convert org_id (could be id or name) into id. org_show = logic.get_action('organization_show') @@ -2645,7 +2672,7 @@ def organization_activity_list(context, data_dict): return model_dictize.activity_list_dictize(activity_objects, context) -@logic.validate(logic.schema.default_pagination_schema) +@logic.validate(logic.schema.default_dashboard_activity_list_schema) def recently_changed_packages_activity_list(context, data_dict): '''Return the activity stream of all recently added or changed packages. @@ -2653,8 +2680,9 @@ def recently_changed_packages_activity_list(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: list of dictionaries @@ -2664,8 +2692,7 @@ def recently_changed_packages_activity_list(context, data_dict): # authorized to read. model = context['model'] offset = data_dict.get('offset', 0) - limit = int( - data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) + limit = data_dict['limit'] # defaulted, limited & made an int by schema _activity_objects = model.activity.recently_changed_packages_activity_list( limit=limit, offset=offset) @@ -2704,8 +2731,9 @@ def user_activity_list_html(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: string @@ -2735,8 +2763,9 @@ def package_activity_list_html(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: string @@ -2766,8 +2795,9 @@ def group_activity_list_html(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: string @@ -2793,6 +2823,14 @@ def organization_activity_list_html(context, data_dict): :param id: the id or name of the organization :type id: string + :param offset: where to start getting activity items from + (optional, default: ``0``) + :type offset: int + :param limit: the maximum number of activities to return + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) + :type limit: int :rtype: string @@ -2821,8 +2859,9 @@ def recently_changed_packages_activity_list_html(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - ckan.activity_list_limit setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :type limit: int :rtype: string @@ -3313,7 +3352,7 @@ def _group_or_org_followee_list(context, data_dict, is_org=False): return [model_dictize.group_dictize(group, context) for group in groups] -@logic.validate(logic.schema.default_pagination_schema) +@logic.validate(logic.schema.default_dashboard_activity_list_schema) def dashboard_activity_list(context, data_dict): '''Return the authorized (via login or API key) user's dashboard activity stream. @@ -3329,8 +3368,9 @@ def dashboard_activity_list(context, data_dict): (optional, default: 0) :type offset: int :param limit: the maximum number of activities to return - (optional, default: 31, the default value is configurable via the - :ref:`ckan.activity_list_limit` setting) + (optional, default: ``31`` unless set in site's configuration + ``ckan.activity_list_limit``, upper limit: ``100`` unless set in + site's configuration ``ckan.activity_list_limit_max``) :rtype: list of activity dictionaries @@ -3340,8 +3380,7 @@ def dashboard_activity_list(context, data_dict): model = context['model'] user_id = model.User.get(context['user']).id offset = data_dict.get('offset', 0) - limit = int( - data_dict.get('limit', config.get('ckan.activity_list_limit', 31))) + limit = data_dict['limit'] # defaulted, limited & made an int by schema # FIXME: Filter out activities whose subject or object the user is not # authorized to read. @@ -3368,7 +3407,7 @@ def dashboard_activity_list(context, data_dict): return activity_dicts -@logic.validate(ckan.logic.schema.default_pagination_schema) +@logic.validate(ckan.logic.schema.default_dashboard_activity_list_schema) def dashboard_activity_list_html(context, data_dict): '''Return the authorized (via login or API key) user's dashboard activity stream as HTML. diff --git a/ckan/logic/schema.py b/ckan/logic/schema.py index fe8e05c38fc..79e76c7362e 100644 --- a/ckan/logic/schema.py +++ b/ckan/logic/schema.py @@ -11,7 +11,10 @@ ignore, if_empty_same_as, not_missing, - ignore_empty + ignore_empty, + default, + configured_default, + limit_to_configured_maximum ) from ckan.logic.converters import (convert_user_name_or_id_to_id, convert_package_name_or_id_to_id, @@ -567,13 +570,20 @@ def default_pagination_schema(): def default_dashboard_activity_list_schema(): schema = default_pagination_schema() - schema['id'] = [unicode] + schema['limit'] = [ + configured_default('ckan.activity_list_limit', 31), + natural_number_validator, + limit_to_configured_maximum('ckan.activity_list_limit_max', 100)] return schema def default_activity_list_schema(): schema = default_pagination_schema() schema['id'] = [not_missing, unicode] + schema['limit'] = [ + configured_default('ckan.activity_list_limit', 31), + natural_number_validator, + limit_to_configured_maximum('ckan.activity_list_limit_max', 100)] return schema @@ -590,7 +600,8 @@ def default_package_search_schema(): 'q': [ignore_missing, unicode], 'fl': [ignore_missing, list_of_strings], 'fq': [ignore_missing, unicode], - 'rows': [ignore_missing, natural_number_validator], + 'rows': [default(10), natural_number_validator, + limit_to_configured_maximum('ckan.search.rows_max', 1000)], 'sort': [ignore_missing, unicode], 'start': [ignore_missing, natural_number_validator], 'qf': [ignore_missing, unicode], diff --git a/ckan/model/group.py b/ckan/model/group.py index aeef561d2c8..cfbacedb408 100644 --- a/ckan/model/group.py +++ b/ckan/model/group.py @@ -116,13 +116,15 @@ class Group(vdm.sqlalchemy.RevisionedObjectMixin, domain_object.DomainObject): def __init__(self, name=u'', title=u'', description=u'', image_url=u'', - type=u'group', approval_status=u'approved'): + type=u'group', approval_status=u'approved', + is_organization=False): self.name = name self.title = title self.description = description self.image_url = image_url self.type = type self.approval_status = approval_status + self.is_organization = is_organization @property def display_name(self): diff --git a/ckan/tests/legacy/functional/api/test_dashboard.py b/ckan/tests/legacy/functional/api/test_dashboard.py index 7482dd1a839..7d09146dced 100644 --- a/ckan/tests/legacy/functional/api/test_dashboard.py +++ b/ckan/tests/legacy/functional/api/test_dashboard.py @@ -12,6 +12,7 @@ import paste import pylons.test from ckan.tests.legacy import CreateTestData +import ckan.tests.helpers as helpers class TestDashboard(object): '''Tests for the logic action functions related to the user's dashboard.''' @@ -309,6 +310,7 @@ def test_07_mark_new_activities_as_read(self): assert self.dashboard_new_activities_count(self.new_user) == 0 assert len(self.dashboard_new_activities(self.new_user)) == 0 + @helpers.change_config('ckan.activity_list_limit', '15') def test_08_maximum_number_of_new_activities(self): '''Test that the new activities count does not go higher than 15, even if there are more than 15 new activities from the user's followers.''' diff --git a/ckan/tests/legacy/functional/test_activity.py b/ckan/tests/legacy/functional/test_activity.py index f40f1397d78..e9eac68fd43 100644 --- a/ckan/tests/legacy/functional/test_activity.py +++ b/ckan/tests/legacy/functional/test_activity.py @@ -6,6 +6,7 @@ import paste.fixture from routes import url_for from nose import SkipTest +import ckan.tests.helpers as helpers import ckan from ckan.logic.action.create import package_create, user_create, group_create @@ -35,7 +36,7 @@ def setup(cls): def teardown(cls): ckan.model.repo.rebuild_db() - + @helpers.change_config('ckan.activity_list_limit', '15') def test_user_activity(self): """Test user activity streams HTML rendering.""" diff --git a/ckan/tests/lib/dictization/test_model_dictize.py b/ckan/tests/lib/dictization/test_model_dictize.py index 43de44788c1..f118fe83309 100644 --- a/ckan/tests/lib/dictization/test_model_dictize.py +++ b/ckan/tests/lib/dictization/test_model_dictize.py @@ -240,6 +240,17 @@ def test_group_dictize_with_package_list_limited_over(self): assert_equal(len(group['packages']), 3) + @helpers.change_config('ckan.search.rows_max', '4') + def test_group_dictize_with_package_list_limited_by_config(self): + group_ = factories.Group() + for _ in range(5): + factories.Dataset(groups=[{'name': group_['name']}]) + group_obj = model.Session.query(model.Group).filter_by().first() + context = {'model': model, 'session': model.Session} + group = model_dictize.group_dictize(group_obj, context) + assert_equal(len(group['packages']), 4) + # limited by ckan.search.rows_max + def test_group_dictize_with_package_count(self): # group_list_dictize calls it like this by default group_ = factories.Group() diff --git a/ckan/tests/logic/action/test_get.py b/ckan/tests/logic/action/test_get.py index fd6fe4ea8ed..e125d5885fa 100644 --- a/ckan/tests/logic/action/test_get.py +++ b/ckan/tests/logic/action/test_get.py @@ -159,6 +159,36 @@ def test_group_list_all_fields(self): assert 'users' not in group_list[0] assert 'datasets' not in group_list[0] + def _create_bulk_groups(self, name, count): + from ckan import model + model.repo.new_revision() + groups = [model.Group(name='{}_{}'.format(name, i)) + for i in range(count)] + model.Session.add_all(groups) + model.repo.commit_and_remove() + + def test_limit_default(self): + self._create_bulk_groups('group_default', 1010) + results = helpers.call_action('group_list') + eq(len(results), 1000) # i.e. default value + + @helpers.change_config('ckan.group_and_organization_list_max', '5') + def test_limit_configured(self): + self._create_bulk_groups('group_default', 7) + results = helpers.call_action('group_list') + eq(len(results), 5) # i.e. configured limit + + def test_all_fields_limit_default(self): + self._create_bulk_groups('org_all_fields_default', 30) + results = helpers.call_action('group_list', all_fields=True) + eq(len(results), 25) # i.e. default value + + @helpers.change_config('ckan.group_and_organization_list_all_fields_max', '5') + def test_all_fields_limit_configured(self): + self._create_bulk_groups('org_all_fields_default', 30) + results = helpers.call_action('group_list', all_fields=True) + eq(len(results), 5) # i.e. configured limit + def test_group_list_extras_returned(self): group = factories.Group(extras=[{'key': 'key1', 'value': 'val1'}]) @@ -389,6 +419,16 @@ def test_group_show_does_not_show_private_datasets(self): in group['packages']], ( "group_show() should never show private datasets") + @helpers.change_config('ckan.search.rows_max', '5') + def test_package_limit_configured(self): + group = factories.Group() + for i in range(7): + factories.Dataset(groups=[{'id': group['id']}]) + id = group['id'] + results = helpers.call_action('group_show', id=id, + include_datasets=1) + eq(len(results['packages']), 5) # i.e. ckan.search.rows_max + class TestOrganizationList(helpers.FunctionalTestBase): @@ -447,6 +487,37 @@ def test_organization_list_return_custom_organization_type(self): assert (sorted(org_list) == sorted([g['name'] for g in [org2]])), '{}'.format(org_list) + def _create_bulk_orgs(self, name, count): + from ckan import model + model.repo.new_revision() + orgs = [model.Group(name='{}_{}'.format(name, i), is_organization=True, + type='organization') + for i in range(count)] + model.Session.add_all(orgs) + model.repo.commit_and_remove() + + def test_limit_default(self): + self._create_bulk_orgs('org_default', 1010) + results = helpers.call_action('organization_list') + eq(len(results), 1000) # i.e. default value + + @helpers.change_config('ckan.group_and_organization_list_max', '5') + def test_limit_configured(self): + self._create_bulk_orgs('org_default', 7) + results = helpers.call_action('organization_list') + eq(len(results), 5) # i.e. configured limit + + def test_all_fields_limit_default(self): + self._create_bulk_orgs('org_all_fields_default', 30) + results = helpers.call_action('organization_list', all_fields=True) + eq(len(results), 25) # i.e. default value + + @helpers.change_config('ckan.group_and_organization_list_all_fields_max', '5') + def test_all_fields_limit_configured(self): + self._create_bulk_orgs('org_all_fields_default', 30) + results = helpers.call_action('organization_list', all_fields=True) + eq(len(results), 5) # i.e. configured limit + class TestOrganizationShow(helpers.FunctionalTestBase): @@ -518,6 +589,16 @@ def test_organization_show_private_packages_not_returned(self): assert org_dict['packages'][0]['name'] == 'dataset_1' assert org_dict['package_count'] == 1 + @helpers.change_config('ckan.search.rows_max', '5') + def test_package_limit_configured(self): + org = factories.Organization() + for i in range(7): + factories.Dataset(owner_org=org['id']) + id = org['id'] + results = helpers.call_action('organization_show', id=id, + include_datasets=1) + eq(len(results['packages']), 5) # i.e. ckan.search.rows_max + class TestUserList(helpers.FunctionalTestBase): @@ -883,6 +964,23 @@ def test_bad_solr_parameter(self): # SOLR error is 'Missing sort order' or 'Missing_sort_order', # depending on the solr version. + def _create_bulk_datasets(self, name, count): + from ckan import model + model.repo.new_revision() + pkgs = [model.Package(name='{}_{}'.format(name, i)) + for i in range(count)] + model.Session.add_all(pkgs) + model.repo.commit_and_remove() + def test_rows_returned_default(self): + self._create_bulk_datasets('rows_default', 11) + results = logic.get_action('package_search')({}, {}) + eq(len(results['results']), 10) # i.e. 'rows' default value + @helpers.change_config('ckan.search.rows_max', '12') + def test_rows_returned_limited(self): + self._create_bulk_datasets('rows_limited', 14) + results = logic.get_action('package_search')({}, {'rows': '15'}) + eq(len(results['results']), 12) # i.e. ckan.search.rows_max + def test_facets(self): org = factories.Organization(name='test-org-facet', title='Test Org') factories.Dataset(owner_org=org['id']) @@ -2199,3 +2297,196 @@ def test_not_existing_job(self): Test showing a not existing job. ''' helpers.call_action(u'job_show', id=u'does-not-exist') + + +class TestPackageActivityList(helpers.FunctionalTestBase): + def _create_bulk_package_activities(self, count): + dataset = factories.Dataset() + from ckan import model + objs = [ + model.Activity( + user_id=None, object_id=dataset['id'], revision_id=None, + activity_type=None, data=None) + for i in range(count)] + model.Session.add_all(objs) + model.repo.commit_and_remove() + return dataset['id'] + + def test_limit_default(self): + id = self._create_bulk_package_activities(35) + results = helpers.call_action('package_activity_list', id=id) + eq(len(results), 31) # i.e. default value + + @helpers.change_config('ckan.activity_list_limit', '5') + def test_limit_configured(self): + id = self._create_bulk_package_activities(7) + results = helpers.call_action('package_activity_list', id=id) + eq(len(results), 5) # i.e. ckan.activity_list_limit + + @helpers.change_config('ckan.activity_list_limit', '5') + @helpers.change_config('ckan.activity_list_limit_max', '7') + def test_limit_hits_max(self): + id = self._create_bulk_package_activities(9) + results = helpers.call_action('package_activity_list', id=id, limit='9') + eq(len(results), 7) # i.e. ckan.activity_list_limit_max + + +class TestUserActivityList(helpers.FunctionalTestBase): + def _create_bulk_user_activities(self, count): + user = factories.User() + from ckan import model + objs = [ + model.Activity( + user_id=user['id'], object_id=None, revision_id=None, + activity_type=None, data=None) + for i in range(count)] + model.Session.add_all(objs) + model.repo.commit_and_remove() + return user['id'] + + def test_limit_default(self): + id = self._create_bulk_user_activities(35) + results = helpers.call_action('user_activity_list', id=id) + eq(len(results), 31) # i.e. default value + + @helpers.change_config('ckan.activity_list_limit', '5') + def test_limit_configured(self): + id = self._create_bulk_user_activities(7) + results = helpers.call_action('user_activity_list', id=id) + eq(len(results), 5) # i.e. ckan.activity_list_limit + + @helpers.change_config('ckan.activity_list_limit', '5') + @helpers.change_config('ckan.activity_list_limit_max', '7') + def test_limit_hits_max(self): + id = self._create_bulk_user_activities(9) + results = helpers.call_action('user_activity_list', id=id, limit='9') + eq(len(results), 7) # i.e. ckan.activity_list_limit_max + + +class TestGroupActivityList(helpers.FunctionalTestBase): + def _create_bulk_group_activities(self, count): + group = factories.Group() + from ckan import model + objs = [ + model.Activity( + user_id=None, object_id=group['id'], revision_id=None, + activity_type=None, data=None) + for i in range(count)] + model.Session.add_all(objs) + model.repo.commit_and_remove() + return group['id'] + + def test_limit_default(self): + id = self._create_bulk_group_activities(35) + results = helpers.call_action('group_activity_list', id=id) + eq(len(results), 31) # i.e. default value + + @helpers.change_config('ckan.activity_list_limit', '5') + def test_limit_configured(self): + id = self._create_bulk_group_activities(7) + results = helpers.call_action('group_activity_list', id=id) + eq(len(results), 5) # i.e. ckan.activity_list_limit + + @helpers.change_config('ckan.activity_list_limit', '5') + @helpers.change_config('ckan.activity_list_limit_max', '7') + def test_limit_hits_max(self): + id = self._create_bulk_group_activities(9) + results = helpers.call_action('group_activity_list', id=id, limit='9') + eq(len(results), 7) # i.e. ckan.activity_list_limit_max + + +class TestOrganizationActivityList(helpers.FunctionalTestBase): + def _create_bulk_org_activities(self, count): + org = factories.Organization() + from ckan import model + objs = [ + model.Activity( + user_id=None, object_id=org['id'], revision_id=None, + activity_type=None, data=None) + for i in range(count)] + model.Session.add_all(objs) + model.repo.commit_and_remove() + return org['id'] + + def test_limit_default(self): + id = self._create_bulk_org_activities(35) + results = helpers.call_action('organization_activity_list', id=id) + eq(len(results), 31) # i.e. default value + + @helpers.change_config('ckan.activity_list_limit', '5') + def test_limit_configured(self): + id = self._create_bulk_org_activities(7) + results = helpers.call_action('organization_activity_list', id=id) + eq(len(results), 5) # i.e. ckan.activity_list_limit + + @helpers.change_config('ckan.activity_list_limit', '5') + @helpers.change_config('ckan.activity_list_limit_max', '7') + def test_limit_hits_max(self): + id = self._create_bulk_org_activities(9) + results = helpers.call_action('organization_activity_list', id=id, limit='9') + eq(len(results), 7) # i.e. ckan.activity_list_limit_max + + +class TestRecentlyChangedPackagesActivityList(helpers.FunctionalTestBase): + def _create_bulk_package_activities(self, count): + from ckan import model + objs = [ + model.Activity( + user_id=None, object_id=None, revision_id=None, + activity_type='new_package', data=None) + for i in range(count)] + model.Session.add_all(objs) + model.repo.commit_and_remove() + + def test_limit_default(self): + self._create_bulk_package_activities(35) + results = helpers.call_action('recently_changed_packages_activity_list') + eq(len(results), 31) # i.e. default value + + @helpers.change_config('ckan.activity_list_limit', '5') + def test_limit_configured(self): + self._create_bulk_package_activities(7) + results = helpers.call_action('recently_changed_packages_activity_list') + eq(len(results), 5) # i.e. ckan.activity_list_limit + + @helpers.change_config('ckan.activity_list_limit', '5') + @helpers.change_config('ckan.activity_list_limit_max', '7') + def test_limit_hits_max(self): + self._create_bulk_package_activities(9) + results = helpers.call_action('recently_changed_packages_activity_list', limit='9') + eq(len(results), 7) # i.e. ckan.activity_list_limit_max + + +class TestDashboardActivityList(helpers.FunctionalTestBase): + def _create_bulk_package_activities(self, count): + user = factories.User() + from ckan import model + objs = [ + model.Activity( + user_id=user['id'], object_id=None, revision_id=None, + activity_type=None, data=None) + for i in range(count)] + model.Session.add_all(objs) + model.repo.commit_and_remove() + return user['id'] + + def test_limit_default(self): + id = self._create_bulk_package_activities(35) + results = helpers.call_action('dashboard_activity_list', + context={'user': id}) + eq(len(results), 31) # i.e. default value + + @helpers.change_config('ckan.activity_list_limit', '5') + def test_limit_configured(self): + id = self._create_bulk_package_activities(7) + results = helpers.call_action('dashboard_activity_list', + context={'user': id}) + eq(len(results), 5) # i.e. ckan.activity_list_limit + + @helpers.change_config('ckan.activity_list_limit', '5') + @helpers.change_config('ckan.activity_list_limit_max', '7') + def test_limit_hits_max(self): + id = self._create_bulk_package_activities(9) + results = helpers.call_action('dashboard_activity_list', limit='9', + context={'user': id}) + eq(len(results), 7) # i.e. ckan.activity_list_limit_max diff --git a/doc/maintaining/configuration.rst b/doc/maintaining/configuration.rst index 9ff1b764080..4b18c38433c 100644 --- a/doc/maintaining/configuration.rst +++ b/doc/maintaining/configuration.rst @@ -713,6 +713,53 @@ Default value: ``None`` List of the extra resource fields that would be used when searching. +.. _ckan.search.rows_max: + +ckan.search.rows_max +^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.search.rows_max = 1000 + +Default value: ``1000`` + +Maximum allowed value for rows returned. Specifically this limits: + +* ``package_search``'s ``rows`` parameter +* ``group_show`` and ``organization_show``'s number of datasets returned when specifying ``include_datasets=true`` + +.. _ckan.group_and_organization_list_max: + +ckan.group_and_organization_list_max +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.group_and_organization_list_max = 1000 + +Default value: ``1000`` + +Maximum number of groups/organizations returned when listing them. Specifically this limits: + +* ``group_list``'s ``limit`` when ``all_fields=false`` +* ``organization_list``'s ``limit`` when ``all_fields=false`` + +.. _ckan.group_and_organization_list_all_fields_max: + +ckan.group_and_organization_list_all_fields_max +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.group_and_organization_list_all_fields_max = 100 + +Default value: ``25`` + +Maximum number of groups/organizations returned when listing them in detail. Specifically this limits: + +* ``group_list``'s ``limit`` when ``all_fields=true`` +* ``organization_list``'s ``limit`` when ``all_fields=true`` Redis Settings --------------- @@ -1497,10 +1544,22 @@ Example:: ckan.activity_list_limit = 31 -Default value: ``infinite`` +Default value: ``31`` + +This controls the number of activities to show in the Activity Stream. + +.. _ckan.activity_list_limit_max: + +ckan.activity_list_limit_max +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + ckan.activity_list_limit_max = 100 -This controls the number of activities to show in the Activity Stream. By default, it shows everything. +Default value: ``100`` +Maximum allowed value for Activity Stream ``limit`` parameter. .. _ckan.email_notifications_since: diff --git a/test-core.ini b/test-core.ini index 43feb67893c..879ee5547b3 100644 --- a/test-core.ini +++ b/test-core.ini @@ -92,8 +92,6 @@ ckan.datasets_per_page = 20 ckan.activity_streams_email_notifications = True -ckan.activity_list_limit = 15 - ckan.tracking_enabled = true beaker.session.key = ckan