From 3fddcd5f8d7083b39ba73ee0ff0d8e41ef4fdb20 Mon Sep 17 00:00:00 2001 From: Ian Murray Date: Tue, 20 Mar 2012 14:27:29 +0000 Subject: [PATCH] [release-v1.6.1] Added pagination to the 'current_package_list_with_resources' action. Using this action to retrieve the whole list of datasets in even a moderately-sized catalogue is extremely slow. (In the order of 5-6 datasets/sec on my development machine). Aside from this being slow, this means that since one of our supported deployments is proxying through nginx, nginx will timeout the request before it is completed. So as a worksround for now, the datasets can be paginated through multiple smaller requests. Meaning at least the connection won't timeout; even if the total time retrieving the datasets is approximately the same. --- ckan/logic/action/get.py | 4 +++- doc/apiv3.rst | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ckan/logic/action/get.py b/ckan/logic/action/get.py index bf28588a769..48913c37194 100644 --- a/ckan/logic/action/get.py +++ b/ckan/logic/action/get.py @@ -62,6 +62,7 @@ def current_package_list_with_resources(context, data_dict): model = context["model"] user = context["user"] limit = data_dict.get("limit") + page = int(data_dict.get('page', 1)) check_access('current_package_list_with_resources', context, data_dict) @@ -71,7 +72,8 @@ def current_package_list_with_resources(context, data_dict): query = query.order_by(model.package_revision_table.c.revision_timestamp.desc()) if limit: - query = query.limit(limit) + query = query.limit(int(limit)) + query = query.offset((page-1)*limit) pack_rev = query.all() return _package_list_with_resources(context, pack_rev) diff --git a/doc/apiv3.rst b/doc/apiv3.rst index 99b1b816f47..c350500d3a4 100644 --- a/doc/apiv3.rst +++ b/doc/apiv3.rst @@ -55,7 +55,7 @@ Logic Action Parameter keys ====================================== =========================== site_read (none) package_list (none) -current_package_list_with_resources limit +current_package_list_with_resources limit, page revision_list (none) package_revision_list id group_list all_fields