Merge pull request #413 from steffenschumacher/#412_manual_pagination

Fixes manual pagination (#412) and argless filter (#386)

pynetbox/core/endpoint.py

@@ -16,7 +16,7 @@
 from pynetbox.core.query import Request, RequestError
 from pynetbox.core.response import Record, RecordSet
 
-RESERVED_KWARGS = ("offset",)
+RESERVED_KWARGS = ()
 
 
 class Endpoint(object):
@@ -71,13 +71,14 @@ def _lookup_ret_obj(self, name, model):
             ret = Record
         return ret
 
-    def all(self, limit=0):
+    def all(self, limit=0, offset=None):
         """Queries the 'ListView' of a given endpoint.
 
         Returns all objects from an endpoint.
 
         :arg int,optional limit: Overrides the max page size on
             paginated returns.
+        :arg int,optional offset: Overrides the offset on paginated returns.
 
         :Returns: A :py:class:`.RecordSet` object.
 
@@ -92,13 +93,16 @@ def all(self, limit=0):
         test1-leaf3
         >>>
         """
+        if limit == 0 and offset is not None:
+            raise ValueError("offset requires a positive limit value")
         req = Request(
             base="{}/".format(self.url),
             token=self.token,
             session_key=self.session_key,
             http_session=self.api.http_session,
             threading=self.api.threading,
             limit=limit,
+            offset=offset,
         )
 
         return RecordSet(self, req)
@@ -180,6 +184,7 @@ def filter(self, *args, **kwargs):
             endpoint accepts can be added as a keyword arg.
         :arg int,optional limit: Overrides the max page size on
             paginated returns.
+        :arg int,optional offset: Overrides the offset on paginated returns.
 
         :Returns: A :py:class:`.RecordSet` object.
 
@@ -231,22 +236,24 @@ def filter(self, *args, **kwargs):
         if args:
             kwargs.update({"q": args[0]})
 
-        if not kwargs:
-            raise ValueError("filter must be passed kwargs. Perhaps use all() instead.")
         if any(i in RESERVED_KWARGS for i in kwargs):
             raise ValueError(
                 "A reserved kwarg was passed ({}). Please remove it "
                 "and try again.".format(RESERVED_KWARGS)
             )
-
+        limit = kwargs.pop("limit") if "limit" in kwargs else 0
+        offset = kwargs.pop("offset") if "offset" in kwargs else None
+        if limit == 0 and offset is not None:
+            raise ValueError("offset requires a positive limit value")
         req = Request(
             filters=kwargs,
             base=self.url,
             token=self.token,
             session_key=self.session_key,
             http_session=self.api.http_session,
             threading=self.api.threading,
-            limit=kwargs.get("limit", 0),
+            limit=limit,
+            offset=offset,
         )
 
         return RecordSet(self, req)

pynetbox/core/query.py

@@ -131,6 +131,7 @@ def __init__(
         http_session,
         filters=None,
         limit=None,
+        offset=None,
         key=None,
         token=None,
         private_key=None,
@@ -151,7 +152,7 @@ def __init__(
                 string.
         """
         self.base = self.normalize_url(base)
-        self.filters = filters
+        self.filters = filters or None
         self.key = key
         self.token = token
         self.private_key = private_key
@@ -160,6 +161,7 @@ def __init__(
         self.url = self.base if not key else "{}{}/".format(self.base, key)
         self.threading = threading
         self.limit = limit
+        self.offset = offset
 
     def get_openapi(self):
         """ Gets the OpenAPI Spec """
@@ -309,10 +311,17 @@ def get(self, add_params=None):
 
         if not add_params and self.limit is not None:
             add_params = {"limit": self.limit}
+            if self.limit and self.offset is not None:
+                # if non-zero limit and some offset -> add offset
+                add_params["offset"] = self.offset
         req = self._make_call(add_params=add_params)
         if isinstance(req, dict) and req.get("results") is not None:
             self.count = req["count"]
-            if self.threading:
+            if self.offset is not None:
+                # only yield requested page results if paginating
+                for i in req["results"]:
+                    yield i
+            elif self.threading:
                 ret = req["results"]
                 if req.get("next"):
                     page_size = len(req["results"])

tests/unit/test_endpoint.py

@@ -22,21 +22,21 @@ def test_filter(self):
             test = test_obj.filter(test="test")
             self.assertEqual(len(test), 2)
 
-    def test_filter_empty_kwargs(self):
+    def test_filter_invalid_pagination_args(self):
 
         api = Mock(base_url="http://localhost:8000/api")
         app = Mock(name="test")
         test_obj = Endpoint(api, app, "test")
         with self.assertRaises(ValueError) as _:
-            test_obj.filter()
+            test_obj.filter(offset=1)
 
-    def test_filter_reserved_kwargs(self):
+    def test_all_invalid_pagination_args(self):
 
         api = Mock(base_url="http://localhost:8000/api")
         app = Mock(name="test")
         test_obj = Endpoint(api, app, "test")
         with self.assertRaises(ValueError) as _:
-            test_obj.filter(offset=1)
+            test_obj.all(offset=1)
 
     def test_choices(self):
         with patch("pynetbox.core.query.Request.options", return_value=Mock()) as mock:

tests/unit/test_query.py

@@ -57,3 +57,31 @@ def test_get_count_no_filters(self):
             headers={"accept": "application/json;"},
             json=None,
         )
+
+    def test_get_manual_pagination(self):
+        test_obj = Request(
+            http_session=Mock(),
+            base="http://localhost:8001/api/dcim/devices",
+            limit=10,
+            offset=20,
+        )
+        test_obj.http_session.get.return_value.json.return_value = {
+            "count": 4,
+            "next": "http://localhost:8001/api/dcim/devices?limit=10&offset=30",
+            "previous": False,
+            "results": [1, 2, 3, 4],
+        }
+        expected = call(
+            "http://localhost:8001/api/dcim/devices/",
+            params={"offset": 20, "limit": 10},
+            headers={"accept": "application/json;"},
+        )
+        test_obj.http_session.get.ok = True
+        generator = test_obj.get()
+        self.assertEqual(len(list(generator)), 4)
+        test_obj.http_session.get.assert_called_with(
+            "http://localhost:8001/api/dcim/devices/",
+            params={"offset": 20, "limit": 10},
+            headers={"accept": "application/json;"},
+            json=None,
+        )