Add support for pagination

Support on all list resources
Updated doc strings
Added section in API doc on pagination

TODO.md

@@ -3,19 +3,14 @@
     - Hostnames and reverse
     - API Tokens for role accounts
     - Setup support for migrations before first preview release
-    - Pagination
     - PATCH support
     - Default Site for User
 
 * Web UI
-    - Form Validation
-    - Subnav Bar
     - Pages
-        * Sites (Update/Delete)
         * Networks (Create/List/Show/Update/Delete)
-        * Network Attributes (Create/List/Show/Update/Delete)
+        * Network Attributes (Show)
         * Permissions (Update)
-        * Changelog
 
 * Python Client Library
     - Everything

docs/api.rst

@@ -50,3 +50,32 @@ will contain an error ``code`` and ``message``.
             "message": "Resource not found."
         }
     }
+
+Pagination
+----------
+
+Most, if not all, responses that return a list of resources will support pagination. If the
+``data`` object on the response has a ``total`` attribute then the endpoint supports pagination.
+When making a request against this endpoint ``limit`` and ``offset`` query parameters are
+supported.
+
+An example response for querying the ``sites`` endpoint might look like:
+
+.. sourcecode:: javascript
+
+    {
+        "status": "ok",
+        "data": {
+            "sites": [
+                {
+                    "id": 1
+                    "name": "Site 1",
+                    "description": ""
+                }
+            ],
+            "limit": null,
+            "offset": 0,
+            "total": 1
+        }
+    }
+

nsot/handlers/api.py

@@ -109,18 +109,30 @@ def get(self):
                             "name": "Site 1",
                             "description": ""
                         }
-                    ]
+                    ],
+                    "limit": null,
+                    "offset": 0,
+                    "total": 1,
                 }
             }
 
         :reqheader X-NSoT-Email: required for all api requests.
 
+        :query int limit: (*optional*) Limit result to N resources.
+        :query int offset: (*optional*) Skip the first N resources.
+
         :statuscode 200: The request was successful.
         :statuscode 401: The request was made without being logged in.
         """
-        sites = self.session.query(models.Site).all()
+        sites = self.session.query(models.Site)
+        offset, limit = self.get_pagination_values()
+        sites, total = self.paginate_query(sites, offset, limit)
+
         self.success({
-            "sites": [site.to_dict() for site in sites],
+            "sites": [site.to_dict() for site in sites.all()],
+            "limit": limit,
+            "offset": offset,
+            "total": total,
         })
 
 
@@ -418,13 +430,18 @@ def get(self, site_id):
                             "description": "",
                             "required": false
                         }
-                    ]
+                    ],
+                    "limit": null,
+                    "offset": 0,
+                    "total": 1,
                 }
             }
 
         :param site_id: ID of the Site to retrieve Network Attributes from.
         :type site_id: int
 
+        :query int limit: (*optional*) Limit result to N resources.
+        :query int offset: (*optional*) Skip the first N resources.
         :query string name: (*optional*) Filter to attribute with name
         :query bool required: (*optional*) Filter to attributes that are required
 
@@ -451,8 +468,14 @@ def get(self, site_id):
         if required:
             attributes = attributes.filter_by(required=True)
 
+        offset, limit = self.get_pagination_values()
+        attributes, total = self.paginate_query(attributes, offset, limit)
+
         self.success({
             "network_attributes": [attribute.to_dict() for attribute in attributes],
+            "limit": limit,
+            "offset": offset,
+            "total": total,
         })
 
 
@@ -810,13 +833,18 @@ def get(self, site_id):
                             "prefix_length": "8",
                             "attributes": {}
                         }
-                    ]
+                    ],
+                    "limit": null,
+                    "offset": 0,
+                    "total": 1,
                 }
             }
 
         :param site_id: ID of the Site to retrieve Networks from.
         :type site_id: int
 
+        :query int limit: (*optional*) Limit result to N resources.
+        :query int offset: (*optional*) Skip the first N resources.
         :query bool root_only: (*optional*) Filter to root networks.
                                Default: false
         :query bool include_networks: (*optional*) Include non-IP networks.
@@ -843,8 +871,14 @@ def get(self, site_id):
             include_networks=include_networks
         )
 
+        offset, limit = self.get_pagination_values()
+        networks, total = self.paginate_query(networks, offset, limit)
+
         self.success({
             "networks": [network.to_dict() for network in networks],
+            "limit": limit,
+            "offset": offset,
+            "total": total,
         })
 
 
@@ -1112,7 +1146,10 @@ def get(self, site_id, network_id):
                             "prefix_length": "24",
                             "attributes": {}
                         }
-                    ]
+                    ],
+                    "limit": null,
+                    "offset": 0,
+                    "total": 1,
                 }
             }
 
@@ -1122,6 +1159,8 @@ def get(self, site_id, network_id):
         :param network_id: ID of the Network we're requesting subnets from.
         :type network_id: int
 
+        :query int limit: (*optional*) Limit result to N resources.
+        :query int offset: (*optional*) Skip the first N resources.
         :query bool direct: (*optional*) Return only direct subnets.
                             Default: false
         :query bool include_networks: (*optional*) Include non-IP networks.
@@ -1160,8 +1199,14 @@ def get(self, site_id, network_id):
             include_ips=include_ips, include_networks=include_networks
         )
 
+        offset, limit = self.get_pagination_values()
+        networks, total = self.paginate_query(networks, offset, limit)
+
         self.success({
             "networks": [network.to_dict() for network in networks],
+            "limit": limit,
+            "offset": offset,
+            "total": total,
         })
 
 
@@ -1198,7 +1243,10 @@ def get(self, site_id, network_id):
                             "prefix_length": "8",
                             "attributes": {}
                         }
-                    ]
+                    ],
+                    "limit": null,
+                    "offset": 0,
+                    "total": 1,
                 }
             }
 
@@ -1208,6 +1256,8 @@ def get(self, site_id, network_id):
         :param network_id: ID of the Network we're requesting supernets from.
         :type network_id: int
 
+        :query int limit: (*optional*) Limit result to N resources.
+        :query int offset: (*optional*) Skip the first N resources.
         :query bool direct: (*optional*) Return only direct supernets.
                             Default: false
 
@@ -1237,8 +1287,14 @@ def get(self, site_id, network_id):
 
         networks = network.supernets(self.session, direct=direct)
 
+        offset, limit = self.get_pagination_values()
+        networks, total = self.paginate_query(networks, offset, limit)
+
         self.success({
             "networks": [network.to_dict() for network in networks],
+            "limit": limit,
+            "offset": offset,
+            "total": total,
         })
 
 
@@ -1285,13 +1341,18 @@ def get(self, site_id):
                                 "description": ""
                             },
                         }
-                    ]
+                    ],
+                    "limit": null,
+                    "offset": 0,
+                    "total": 1,
                 }
             }
 
         :param site_id: ID of the Site to retrieve Changes from.
         :type site_id: int
 
+        :query int limit: (*optional*) Limit result to N resources.
+        :query int offset: (*optional*) Skip the first N resources.
         :query string event: (*optional*) Filter result to specific event.
                              Default: false
         :query string resource_type: (*optional*) Filter result to specific
@@ -1338,8 +1399,14 @@ def get(self, site_id):
 
         changes = changes.order_by(desc("change_at"))
 
+        offset, limit = self.get_pagination_values()
+        changes, total = self.paginate_query(changes, offset, limit)
+
         self.success({
             "changes": [change.to_dict() for change in changes],
+            "limit": limit,
+            "offset": offset,
+            "total": total,
         })
 
 
@@ -1440,22 +1507,35 @@ def get(self):
                             "id": 1
                             "email": "user@localhost"
                         }
-                    ]
+                    ],
+                    "limit": null,
+                    "offset": 0,
+                    "total": 1,
                 }
             }
 
         :reqheader X-NSoT-Email: required for all api requests.
 
+        :query int limit: (*optional*) Limit result to N resources.
+        :query int offset: (*optional*) Skip the first N resources.
+
         :statuscode 200: The request was successful.
         :statuscode 401: The request was made without being logged in.
         """
 
         users = self.session.query(models.User)
 
+        offset, limit = self.get_pagination_values()
+        users, total = self.paginate_query(users, offset, limit)
+
         self.success({
             "users": [user.to_dict() for user in users],
+            "limit": limit,
+            "offset": offset,
+            "total": total,
         })
 
+
 class UserHandler(ApiHandler):
     def get(self, user_id):
         """ **Get a specific User**

nsot/handlers/util.py

@@ -100,6 +100,27 @@ def jbody(self):
                 self._jbody = {}
         return self._jbody
 
+    def get_pagination_values(self, max_limit=None):
+        offset = int(self.get_argument("offset", 0))
+        limit = self.get_argument("limit", None)
+        if limit is None:
+            return offset, limit
+
+        limit = int(limit)
+        if max_limit is not None and limit > max_limit:
+            limit = max_limit
+
+        return offset, limit
+
+    def paginate_query(self, query, offset, limit):
+        total = query.count()
+
+        query = query.offset(offset)
+        if limit is not None:
+            query = query.limit(limit)
+
+        return query, total
+
     def prepare(self):
         rv = BaseHandler.prepare(self)
         if rv is not None:

nsot/models.py

@@ -323,8 +323,7 @@ def networks(self, include_networks=True, include_ips=False, root=False,
         if root:
             query = query.filter(Network.parent_id == None)
 
-        networks = query.all()
-        return networks
+        return query
 
 
     @validates("name")

tests/api_tests/test_auth.py

@@ -17,7 +17,7 @@ def test_valid_user(tornado_server):
     assert_success(requests.get(
         "http://localhost:{}/api/sites".format(tornado_server.port),
         headers={"X-NSoT-Email": "gary@localhost"}
-    ), {"sites": []})
+    ), {"sites": [], "limit": None, "offset": 0, "total": 0})
 
 
 def test_invalid_user(tornado_server):

tests/api_tests/test_network_attributes.py

@@ -28,12 +28,17 @@ def test_creation(tornado_server):
     # Successfully get all Network Attributes
     assert_success(
         client.get("/sites/1/network_attributes"),
-        {"network_attributes": [
-            {
-                "id": 1, "name": "attr1", "description": "",
-                "required": False, "site_id": 1
-            },
-        ]}
+        {
+            "network_attributes": [
+                {
+                    "id": 1, "name": "attr1", "description": "",
+                    "required": False, "site_id": 1
+                },
+            ],
+            "limit": None,
+            "offset": 0,
+            "total": 1,
+        }
 
     )