Implement Pager for auto-paging requests

samples/pagination_sample.py

@@ -17,37 +17,6 @@
 import tableauserverclient as TSC
 
 
-class pagination_generator(object):
-    """ This class returns a generator that will iterate over all of the results.
-
-    server is the server object that will be used when calling the callback.  It will be passed
-    to the callback on each iteration
-
-    Callback is expected to take a server object and a request options and return two values, an array of results,
-    and the pagination item from the current call.  This will be used to build subsequent requests.
-    """
-
-    def __init__(self, fetch_more):
-        self._fetch_more = fetch_more
-
-    def __call__(self):
-        current_item_list, last_pagination_item = self._fetch_more(None)  # Prime the generator
-        count = 0
-
-        while count < last_pagination_item.total_available:
-            if len(current_item_list) == 0:
-                current_item_list, last_pagination_item = self._load_next_page(current_item_list, last_pagination_item)
-
-            yield current_item_list.pop(0)
-            count += 1
-
-    def _load_next_page(self, current_item_list, last_pagination_item):
-        next_page = last_pagination_item.page_number + 1
-        opts = TSC.RequestOptions(pagenumber=next_page, pagesize=last_pagination_item.page_size)
-        current_item_list, last_pagination_item = self._fetch_more(opts)
-        return current_item_list, last_pagination_item
-
-
 def main():
 
     parser = argparse.ArgumentParser(description='Return a list of all of the workbooks on your server')
@@ -70,10 +39,32 @@ def main():
     server = TSC.Server(args.server)
 
     with server.auth.sign_in(tableau_auth):
-        generator = pagination_generator(server.workbooks.get)
+
+        # Pager returns a generator that yields one item at a time fetching
+        # from Server only when necessary. Pager takes a server Endpoint as its
+        # first parameter. It will call 'get' on that endpoint. To get workbooks
+        # pass `server.workbooks`, to get users pass` server.users`, etc
+        # You can then loop over the generator to get the objects one at a time
+        # Here we print the workbook id for each workbook
+
         print("Your server contains the following workbooks:\n")
-        for wb in generator():
+        for wb in TSC.Pager(server.workbooks):
             print(wb.name)
 
+        # Pager can also be used in list comprehensions or generator expressions
+        # for compactness and easy filtering. Generator expressions will use less
+        # memory than list comprehsnsions. Consult the Python laguage documentation for
+        # best practices on which are best for your use case. Here we loop over the
+        # Pager and only keep workbooks where the name starts with the letter 'a'
+        # >>> [wb for wb in TSC.Pager(server.workbooks) if wb.name.startswith('a')] # List Comprehension
+        # >>> (wb for wb in TSC.Pager(server.workbooks) if wb.name.startswith('a')) # Generator Expression
+
+        # Since Pager is a generator it follows the standard conventions and can
+        # be fed to a list if you really need all the workbooks in memory at once.
+        # If you need everything, it may be faster to use a larger page size
+
+        # >>> request_options = TSC.RequestOptions(pagesize=1000)
+        # >>> all_workbooks = list(TSC.Pager(server.workbooks, request_options))
+
 if __name__ == '__main__':
     main()

tableauserverclient/__init__.py

@@ -4,7 +4,7 @@
     SiteItem, TableauAuth, UserItem, ViewItem, WorkbookItem, UnpopulatedPropertyError, \
     HourlyInterval, DailyInterval, WeeklyInterval, MonthlyInterval, IntervalItem
 from .server import RequestOptions, Filter, Sort, Server, ServerResponseError,\
-    MissingRequiredFieldError, NotSignedInError
+    MissingRequiredFieldError, NotSignedInError, Pager
 
 __version__ = '0.0.1'
 __VERSION__ = __version__

tableauserverclient/server/__init__.py

@@ -8,4 +8,5 @@
 from .endpoint import Auth, Datasources, Endpoint, Groups, Projects, Schedules, \
     Sites, Users, Views, Workbooks, ServerResponseError, MissingRequiredFieldError
 from .server import Server
+from .pager import Pager
 from .exceptions import NotSignedInError

tableauserverclient/server/pager.py

@@ -0,0 +1,43 @@
+from . import RequestOptions
+
+
+class Pager(object):
+    """
+    Generator that takes an endpoint with `.get` and lazily loads items from Server.
+    Supports all `RequestOptions` including starting on any page.
+    """
+
+    def __init__(self, endpoint, request_opts=None):
+        self._endpoint = endpoint.get
+        self._options = request_opts
+
+        # If we have options we could be starting on any page, backfill the count
+        if self._options:
+            self._count = ((self._options.pagenumber - 1) * self._options.pagesize)
+        else:
+            self._count = 0
+
+    def __iter__(self):
+        # Fetch the first page
+        current_item_list, last_pagination_item = self._endpoint(self._options)
+
+        # Get the rest on demand as a generator
+        while self._count < last_pagination_item.total_available:
+            if len(current_item_list) == 0:
+                current_item_list, last_pagination_item = self._load_next_page(last_pagination_item)
+
+            try:
+                yield current_item_list.pop(0)
+                self._count += 1
+
+            except IndexError:
+                # The total count on Server changed while fetching exit gracefully
+                raise StopIteration
+
+    def _load_next_page(self, last_pagination_item):
+        next_page = last_pagination_item.page_number + 1
+        opts = RequestOptions(pagenumber=next_page, pagesize=last_pagination_item.page_size)
+        if self._options is not None:
+            opts.sort, opts.filter = self._options.sort, self._options.filter
+        current_item_list, last_pagination_item = self._endpoint(opts)
+        return current_item_list, last_pagination_item

tableauserverclient/server/server.py

@@ -1,5 +1,6 @@
 from .exceptions import NotSignedInError
 from .endpoint import Sites, Views, Users, Groups, Workbooks, Datasources, Projects, Auth, Schedules, ServerInfo
+
 import requests
 
 

test/assets/workbook_get_page_1.xml

@@ -0,0 +1,11 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
+    <pagination pageNumber="1" pageSize="1" totalAvailable="3" />
+    <workbooks>
+        <workbook id="6d13b0ca-043d-4d42-8c9d-3f3313ea3a00" name="Page1Workbook" contentUrl="Page1Workbook" showTabs="false" size="1" createdAt="2016-08-03T20:34:04Z" updatedAt="2016-08-04T17:56:41Z">
+            <project id="ee8c6e70-43b6-11e6-af4f-f7b0d8e20760" name="default" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+            <tags />
+        </workbook>
+    </workbooks>
+</tsResponse>

test/assets/workbook_get_page_2.xml

@@ -0,0 +1,14 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
+    <pagination pageNumber="2" pageSize="1" totalAvailable="3" />
+    <workbooks>
+        <workbook id="3cc6cd06-89ce-4fdc-b935-5294135d6d42" name="Page2Workbook" contentUrl="Page2Workbook" showTabs="false" size="26" createdAt="2016-07-26T20:34:56Z" updatedAt="2016-07-26T20:35:05Z">
+            <project id="ee8c6e70-43b6-11e6-af4f-f7b0d8e20760" name="default" />
+            <owner id="5de011f8-5aa9-4d5b-b991-f462c8dd6bb7" />
+            <tags>
+                <tag label="Safari" />
+                <tag label="Sample" />
+            </tags>
+        </workbook>
+    </workbooks>
+</tsResponse>

test/assets/workbook_get_page_3.xml

@@ -0,0 +1,10 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<tsResponse xmlns="http://tableau.com/api" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://tableau.com/api http://tableau.com/api/ts-api-2.3.xsd">
+    <pagination pageNumber="3" pageSize="1" totalAvailable="3" />
+    <workbooks>
+        <workbook id="0413f2d6-387d-4fb6-9823-370d67b1276f" name="Page3Workbook" contentUrl="Page3Workbook" showTabs="false" size="26" createdAt="2016-07-26T20:34:56Z" updatedAt="2016-07-26T20:35:05Z">
+            <project id="63ba565c-f352-41f4-87f5-ba4573fc7c2c" name="default" />
+            <owner id="299fe064-51a5-4e11-93a1-76116239f082" />
+        </workbook>
+    </workbooks>
+</tsResponse>

test/test_pager.py

@@ -0,0 +1,88 @@
+import unittest
+import os
+import requests_mock
+import tableauserverclient as TSC
+
+TEST_ASSET_DIR = os.path.join(os.path.dirname(__file__), 'assets')
+
+GET_XML_PAGE1 = os.path.join(TEST_ASSET_DIR, 'workbook_get_page_1.xml')
+GET_XML_PAGE2 = os.path.join(TEST_ASSET_DIR, 'workbook_get_page_2.xml')
+GET_XML_PAGE3 = os.path.join(TEST_ASSET_DIR, 'workbook_get_page_3.xml')
+
+
+class PagerTests(unittest.TestCase):
+    def setUp(self):
+        self.server = TSC.Server('http://test')
+
+        # Fake sign in
+        self.server._site_id = 'dad65087-b08b-4603-af4e-2887b8aafc67'
+        self.server._auth_token = 'j80k54ll2lfMZ0tv97mlPvvSCRyD0DOM'
+
+        self.baseurl = self.server.workbooks.baseurl
+
+    def test_pager_with_no_options(self):
+        with open(GET_XML_PAGE1, 'rb') as f:
+            page_1 = f.read().decode('utf-8')
+        with open(GET_XML_PAGE2, 'rb') as f:
+            page_2 = f.read().decode('utf-8')
+        with open(GET_XML_PAGE3, 'rb') as f:
+            page_3 = f.read().decode('utf-8')
+        with requests_mock.mock() as m:
+            # Register Pager with default request options
+            m.get(self.baseurl, text=page_1)
+
+            # Register Pager with some pages
+            m.get(self.baseurl + "?pageNumber=1&pageSize=1", text=page_1)
+            m.get(self.baseurl + "?pageNumber=2&pageSize=1", text=page_2)
+            m.get(self.baseurl + "?pageNumber=3&pageSize=1", text=page_3)
+
+            # No options should get all 3
+            workbooks = list(TSC.Pager(self.server.workbooks))
+            self.assertTrue(len(workbooks) == 3)
+
+            # Let's check that workbook items aren't duplicates
+            wb1, wb2, wb3 = workbooks
+            self.assertEqual(wb1.name, 'Page1Workbook')
+            self.assertEqual(wb2.name, 'Page2Workbook')
+            self.assertEqual(wb3.name, 'Page3Workbook')
+
+    def test_pager_with_options(self):
+        with open(GET_XML_PAGE1, 'rb') as f:
+            page_1 = f.read().decode('utf-8')
+        with open(GET_XML_PAGE2, 'rb') as f:
+            page_2 = f.read().decode('utf-8')
+        with open(GET_XML_PAGE3, 'rb') as f:
+            page_3 = f.read().decode('utf-8')
+        with requests_mock.mock() as m:
+            # Register Pager with some pages
+            m.get(self.baseurl + "?pageNumber=1&pageSize=1", text=page_1)
+            m.get(self.baseurl + "?pageNumber=2&pageSize=1", text=page_2)
+            m.get(self.baseurl + "?pageNumber=3&pageSize=1", text=page_3)
+            m.get(self.baseurl + "?pageNumber=1&pageSize=3", text=page_1)
+
+            # Starting on page 2 should get 2 out of 3
+            opts = TSC.RequestOptions(2, 1)
+            workbooks = list(TSC.Pager(self.server.workbooks, opts))
+            self.assertTrue(len(workbooks) == 2)
+
+            # Check that the workbooks are the 2 we think they should be
+            wb2, wb3 = workbooks
+            self.assertEqual(wb2.name, 'Page2Workbook')
+            self.assertEqual(wb3.name, 'Page3Workbook')
+
+            # Starting on 1 with pagesize of 3 should get all 3
+            opts = TSC.RequestOptions(1, 3)
+            workbooks = list(TSC.Pager(self.server.workbooks, opts))
+            self.assertTrue(len(workbooks) == 3)
+            wb1, wb2, wb3 = workbooks
+            self.assertEqual(wb1.name, 'Page1Workbook')
+            self.assertEqual(wb2.name, 'Page2Workbook')
+            self.assertEqual(wb3.name, 'Page3Workbook')
+
+            # Starting on 3 with pagesize of 1 should get the last item
+            opts = TSC.RequestOptions(3, 1)
+            workbooks = list(TSC.Pager(self.server.workbooks, opts))
+            self.assertTrue(len(workbooks) == 1)
+            # Should have the last workbook
+            wb3 = workbooks.pop()
+            self.assertEqual(wb3.name, 'Page3Workbook')