[#4484] manually backported to ckan 2.7.x

ckan/controllers/api.py

@@ -581,6 +581,14 @@ def search(self, ver=None, register=None):
                     # the search
                     if 'callback' in params:
                         del params['callback']
+
+                    # I needed to add limit here, because it bypasses the logic
+                    # function and therefore the validator (which now does the
+                    # limiting when calling package_search)
+                    params['rows'] = min(
+                        int(params.get('rows', 10)),
+                        int(config.get('ckan.search.rows_max', 1000)))
+
                     results = query.run(params)
                 return self._finish_ok(results)
             except search.SearchError, e:

ckan/lib/dictization/model_dictize.py

@@ -389,11 +389,12 @@ def get_packages_for_this_group(group_, just_the_count=False):
                     q['include_private'] = True
 
             if not just_the_count:
-                # Is there a packages limit in the context?
+                # package_search limits 'rows' anyway, so this is only if you
+                # want even fewer
                 try:
                     packages_limit = context['limits']['packages']
                 except KeyError:
-                    q['rows'] = 1000  # Only the first 1000 datasets are returned
+                    del q['rows']  # leave it to package_search to limit it
                 else:
                     q['rows'] = packages_limit
 

ckan/lib/navl/validators.py

@@ -2,7 +2,7 @@
 
 import ckan.lib.navl.dictization_functions as df
 
-from ckan.common import _
+from ckan.common import _, config
 
 missing = df.missing
 StopOnError = df.StopOnError
@@ -81,6 +81,15 @@ def callable(key, data, errors, context):
 
     return callable
 
+def configured_default(config_name, default_value_if_not_configured):
+    '''When key is missing or value is an empty string or None, replace it with
+    a default value from config, or if that isn't set from the
+    default_value_if_not_configured.'''
+    default_value = config.get(config_name)
+    if default_value is None:
+        default_value = default_value_if_not_configured
+    return default(default_value)
+
 def ignore_missing(key, data, errors, context):
     '''If the key is missing from the data, ignore the rest of the key's
     schema.
@@ -123,3 +132,16 @@ def unicode_only(value):
     if not isinstance(value, unicode):
         raise Invalid(_('Must be a Unicode string value'))
     return value
+
+def limit_to_configured_maximum(config_option, default_limit):
+    '''
+    If the value is over a limit, it changes it to the limit. The limit is
+    defined by a configuration option, or if that is not set, a given int
+    default_limit.
+    '''
+    def callable(key, data, errors, context):
+        value = data.get(key)
+        limit = int(config.get(config_option, default_limit))
+        if value > limit:
+            data[key] = limit
+    return callable

ckan/lib/search/query.py

@@ -312,7 +312,9 @@ def run(self, query, permission_labels=None, **kwargs):
             query['q'] = "*:*"
 
         # number of results
-        rows_to_return = min(1000, int(query.get('rows', 10)))
+        rows_to_return = int(query.get('rows', 10))
+        # query['rows'] should be a defaulted int, due to schema, but make
+        # certain, for legacy tests
         if rows_to_return > 0:
             # #1683 Work around problem of last result being out of order
             #       in SOLR 1.4

ckan/logic/action/get.py

@@ -341,6 +341,15 @@ def _group_or_org_list(context, data_dict, is_org=False):
 
     all_fields = asbool(data_dict.get('all_fields', None))
 
+    if all_fields:
+        # all_fields is really computationally expensive, so need a tight limit
+        max_limit = config.get(
+            'ckan.group_and_organization_list_all_fields_max', 25)
+    else:
+        max_limit = config.get('ckan.group_and_organization_list_max', 1000)
+    if limit is None or limit > max_limit:
+        limit = max_limit
+
     # order_by deprecated in ckan 1.8
     # if it is supplied and sort isn't use order_by and raise a warning
     order_by = data_dict.get('order_by', '')
@@ -437,6 +446,11 @@ def group_list(context, data_dict):
         "name asc" string of field name and sort-order. The allowed fields are
         'name', 'package_count' and 'title'
     :type sort: string
+        :param limit: the maximum number of groups returned (optional)
+        Default: ``1000`` when all_fields=false unless set in site's
+        configuration ``ckan.group_and_organization_list_max``
+        Default: ``25`` when all_fields=true unless set in site's
+        configuration ``ckan.group_and_organization_list_all_fields_max``
     :param limit: if given, the list of groups will be broken into pages of
         at most ``limit`` groups per page and only one page will be returned
         at a time (optional)
@@ -486,6 +500,11 @@ def organization_list(context, data_dict):
         "name asc" string of field name and sort-order. The allowed fields are
         'name', 'package_count' and 'title'
     :type sort: string
+    :param limit: the maximum number of organizations returned (optional)
+        Default: ``1000`` when all_fields=false unless set in site's
+        configuration ``ckan.group_and_organization_list_max``
+        Default: ``25`` when all_fields=true unless set in site's
+        configuration ``ckan.group_and_organization_list_all_fields_max``
     :param limit: if given, the list of organizations will be broken into pages
         of at most ``limit`` organizations per page and only one page will be
         returned at a time (optional)
@@ -1739,8 +1758,9 @@ def package_search(context, data_dict):
         documentation, this is a comma-separated string of field names and
         sort-orderings.
     :type sort: string
-    :param rows: the number of matching rows to return. There is a hard limit
-        of 1000 datasets per query.
+    :param rows: the maximum number of matching rows (datasets) to return.
+        (optional, default: ``10``, upper limit: ``1000`` unless set in
+        site's configuration ``ckan.search.rows_max``)
     :type rows: int
     :param start: the offset in the complete result for where the set of
         returned datasets should begin.
@@ -2500,8 +2520,9 @@ def user_activity_list(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: list of dictionaries
@@ -2519,8 +2540,7 @@ def user_activity_list(context, data_dict):
         raise logic.NotFound
 
     offset = data_dict.get('offset', 0)
-    limit = int(
-        data_dict.get('limit', config.get('ckan.activity_list_limit', 31)))
+    limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
     _activity_objects = model.activity.user_activity_list(user.id, limit=limit,
             offset=offset)
@@ -2542,8 +2562,9 @@ def package_activity_list(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: list of dictionaries
@@ -2561,8 +2582,7 @@ def package_activity_list(context, data_dict):
         raise logic.NotFound
 
     offset = int(data_dict.get('offset', 0))
-    limit = int(
-        data_dict.get('limit', config.get('ckan.activity_list_limit', 31)))
+    limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
     _activity_objects = model.activity.package_activity_list(package.id,
             limit=limit, offset=offset)
@@ -2584,8 +2604,9 @@ def group_activity_list(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: list of dictionaries
@@ -2598,8 +2619,7 @@ def group_activity_list(context, data_dict):
     model = context['model']
     group_id = data_dict.get('id')
     offset = data_dict.get('offset', 0)
-    limit = int(
-        data_dict.get('limit', config.get('ckan.activity_list_limit', 31)))
+    limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
     # Convert group_id (could be id or name) into id.
     group_show = logic.get_action('group_show')
@@ -2619,6 +2639,14 @@ def organization_activity_list(context, data_dict):
 
     :param id: the id or name of the organization
     :type id: string
+    :param offset: where to start getting activity items from
+        (optional, default: ``0``)
+    :type offset: int
+    :param limit: the maximum number of activities to return
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
+    :type limit: int
 
     :rtype: list of dictionaries
 
@@ -2630,8 +2658,7 @@ def organization_activity_list(context, data_dict):
     model = context['model']
     org_id = data_dict.get('id')
     offset = data_dict.get('offset', 0)
-    limit = int(
-        data_dict.get('limit', config.get('ckan.activity_list_limit', 31)))
+    limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
     # Convert org_id (could be id or name) into id.
     org_show = logic.get_action('organization_show')
@@ -2645,16 +2672,17 @@ def organization_activity_list(context, data_dict):
     return model_dictize.activity_list_dictize(activity_objects, context)
 
 
-@logic.validate(logic.schema.default_pagination_schema)
+@logic.validate(logic.schema.default_dashboard_activity_list_schema)
 def recently_changed_packages_activity_list(context, data_dict):
     '''Return the activity stream of all recently added or changed packages.
 
     :param offset: where to start getting activity items from
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: list of dictionaries
@@ -2664,8 +2692,7 @@ def recently_changed_packages_activity_list(context, data_dict):
     # authorized to read.
     model = context['model']
     offset = data_dict.get('offset', 0)
-    limit = int(
-        data_dict.get('limit', config.get('ckan.activity_list_limit', 31)))
+    limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
     _activity_objects = model.activity.recently_changed_packages_activity_list(
             limit=limit, offset=offset)
@@ -2704,8 +2731,9 @@ def user_activity_list_html(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: string
@@ -2735,8 +2763,9 @@ def package_activity_list_html(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: string
@@ -2766,8 +2795,9 @@ def group_activity_list_html(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: string
@@ -2793,6 +2823,14 @@ def organization_activity_list_html(context, data_dict):
 
     :param id: the id or name of the organization
     :type id: string
+    :param offset: where to start getting activity items from
+        (optional, default: ``0``)
+    :type offset: int
+    :param limit: the maximum number of activities to return
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
+    :type limit: int
 
     :rtype: string
 
@@ -2821,8 +2859,9 @@ def recently_changed_packages_activity_list_html(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        ckan.activity_list_limit setting)
+       (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
     :type limit: int
 
     :rtype: string
@@ -3313,7 +3352,7 @@ def _group_or_org_followee_list(context, data_dict, is_org=False):
     return [model_dictize.group_dictize(group, context) for group in groups]
 
 
-@logic.validate(logic.schema.default_pagination_schema)
+@logic.validate(logic.schema.default_dashboard_activity_list_schema)
 def dashboard_activity_list(context, data_dict):
     '''Return the authorized (via login or API key) user's dashboard activity
        stream.
@@ -3329,8 +3368,9 @@ def dashboard_activity_list(context, data_dict):
         (optional, default: 0)
     :type offset: int
     :param limit: the maximum number of activities to return
-        (optional, default: 31, the default value is configurable via the
-        :ref:`ckan.activity_list_limit` setting)
+        (optional, default: ``31`` unless set in site's configuration
+        ``ckan.activity_list_limit``, upper limit: ``100`` unless set in
+        site's configuration ``ckan.activity_list_limit_max``)
 
     :rtype: list of activity dictionaries
 
@@ -3340,8 +3380,7 @@ def dashboard_activity_list(context, data_dict):
     model = context['model']
     user_id = model.User.get(context['user']).id
     offset = data_dict.get('offset', 0)
-    limit = int(
-        data_dict.get('limit', config.get('ckan.activity_list_limit', 31)))
+    limit = data_dict['limit']  # defaulted, limited & made an int by schema
 
     # FIXME: Filter out activities whose subject or object the user is not
     # authorized to read.
@@ -3368,7 +3407,7 @@ def dashboard_activity_list(context, data_dict):
     return activity_dicts
 
 
-@logic.validate(ckan.logic.schema.default_pagination_schema)
+@logic.validate(ckan.logic.schema.default_dashboard_activity_list_schema)
 def dashboard_activity_list_html(context, data_dict):
     '''Return the authorized (via login or API key) user's dashboard activity
        stream as HTML.