Add continuation token to list status api (#279)

* Add continuation token to for liststatus * Updated history and version
Azure · Mar 6, 2019 · d56057a · d56057a
1 parent 1a73c66
commit d56057a
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 10 deletions.
diff --git a/HISTORY.rst b/HISTORY.rst
@@ -3,6 +3,11 @@
 Release History
 ===============
 
+0.0.44 (2019-03-05)
++++++++++++++++++++
+* Add continuation token to LISTSTATUS api call
+* Update api-version to 2018-09-01
+
 0.0.43 (2019-03-01)
 +++++++++++++++++++
 * Fix bug in downloader when glob returns a single file

diff --git a/azure/datalake/store/__init__.py b/azure/datalake/store/__init__.py
@@ -6,7 +6,7 @@
 # license information.
 # --------------------------------------------------------------------------
 
-__version__ = "0.0.43"
+__version__ = "0.0.44"
 
 from .core import AzureDLFileSystem
 from .multithread import ADLDownloader

diff --git a/azure/datalake/store/core.py b/azure/datalake/store/core.py
@@ -53,7 +53,7 @@ class AzureDLFileSystem(object):
     url_suffix: str (None)
         Domain to send REST requests to. The end-point URL is constructed
         using this and the store_name. If None, use default.
-    api_version: str (2018-05-01)
+    api_version: str (2018-09-01)
         The API version to target with requests. Changing this value will
         change the behavior of the requests, and can cause unexpected behavior or
         breaking changes. Changes to this value should be undergone with caution.
@@ -118,14 +118,16 @@ def _ls_batched(self, path, batch_size=4000):
             raise ValueError("Batch size must be strictly greater than 1")
         parms = {'listSize': batch_size}
         ret = []
-        data = [None]
+        continuation_token = "NonEmptyStringSentinel"
 
-        while data:
-            data = self.azure.call('LISTSTATUS', path, **parms)['FileStatuses']['FileStatus']
+        while continuation_token != "":
+            ls_call_result = self.azure.call('LISTSTATUS', path, **parms)
+
+            data = ls_call_result['FileStatuses']['FileStatus']
             ret.extend(data)
-            if len(data) < batch_size:
-                break
-            parms['listAfter'] = ret[-1]['pathSuffix']  # Last path to be used as ListAfter
+
+            continuation_token = ls_call_result['FileStatuses']['continuationToken']
+            parms['listAfter'] = continuation_token  # continuationToken to be used as ListAfter
 
         return ret
 

diff --git a/azure/datalake/store/lib.py b/azure/datalake/store/lib.py
@@ -221,7 +221,7 @@ class DatalakeRESTInterface:
     url_suffix: str (None)
         Domain to send REST requests to. The end-point URL is constructed
         using this and the store_name. If None, use default.
-    api_version: str (2018-05-01)
+    api_version: str (2018-09-01)
         The API version to target with requests. Changing this value will
         change the behavior of the requests, and can cause unexpected behavior or
         breaking changes. Changes to this value should be undergone with caution.
@@ -256,7 +256,7 @@ class DatalakeRESTInterface:
     }
 
     def __init__(self, store_name=default_store, token=None,
-                 url_suffix=default_adls_suffix, api_version='2018-05-01', **kwargs):
+                 url_suffix=default_adls_suffix, api_version='2018-09-01', **kwargs):
         # in the case where an empty string is passed for the url suffix, it must be replaced with the default.
         url_suffix = url_suffix or default_adls_suffix
         self.local = threading.local()