Merge branch 'release/v0.5.6'

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 2017-10-20 v0.5.6
+- FEATURE: Added `max_recs` parameter to `Bulk.get_export_data()` which limits the count of records which will be returned
+- CLEANUP: refactored a few small lines of code
+
 ## 2017-10-02 v0.5.5
 - FEATURE: add system fields for bulk activity Bounceback which were made available in the 491 release
 - BUGFIX: auto-set filter on `ActivityType` if exporting an activity

pyeloqua/bulk.py

@@ -394,7 +394,7 @@ def add_options(self, **kwargs):
         validation will result in a 400 status code
         """
 
-        for opt in kwargs.keys():
+        for opt in kwargs:
             self.job['options'][opt] = kwargs[opt]
 
     def add_syncaction(self, action, destination=None, status=None):
@@ -468,7 +468,7 @@ def create_def(self, name):
             'fields': {}
         }
 
-        if len(self.job['filters']) > 0:
+        if self.job['filters']:
             req_data['filter'] = 'AND'.join(self.job['filters'])
 
         for field in self.job['fields']:
@@ -477,7 +477,7 @@ def create_def(self, name):
             else:
                 req_data['fields'][field['name']] = field['statement']
 
-        if len(self.job['options']) > 0:
+        if self.job['options']:
             for option in self.job['options'].keys():
                 req_data[option] = self.job['options'][option]
 
@@ -584,26 +584,26 @@ def post_data(self, data):
 
         _elq_error_(req)
 
-    def get_data(self, endpoint):
+    def get_data(self, endpoint, max_recs=None):
         """
         get data from a given endpoint
         this simplifies the looping process that would otherwise be repeated
 
         :param str endpoint: endpoint to be appended to bulk_base
-        :param bool csv: whether or not to return data as CSV
+        :param int max_recs: Max number of records to return
         """
 
-        url_base = self.bulk_base + endpoint + '?limit=1000&offset={offset}'
+        url_base = self.bulk_base + endpoint + '?limit={limit}&offset={offset}'
 
-        has_more = True
+        limit, has_more = self._set_limit(max_recs, 0, True)
 
         offset = 0
 
         return_data = []
 
         while has_more:
 
-            url = url_base.format(offset=offset)
+            url = url_base.format(limit=limit, offset=offset)
 
             req = requests.get(url=url, auth=self.auth)
 
@@ -614,21 +614,43 @@ def get_data(self, endpoint):
 
             offset += 1000
 
-            has_more = req.json()['hasMore']
+            if max_recs is None:
+
+                has_more = req.json()['hasMore']
+
+            limit, has_more = self._set_limit(max_recs, len(return_data),
+                                              req.json()['hasMore'])
 
         return return_data
 
-    def get_export_data(self):
+
+    def _set_limit(self, max_recs, row_ct, has_more):
+        """  """
+
+        # default limit return 1000
+        # Used when pulling all data
+        if max_recs is None:
+            return 1000, has_more
+
+        if (max_recs - row_ct) >= 1000:
+            return 1000, True
+
+        return (max_recs - row_ct), False
+
+
+    def get_export_data(self, max_recs=None):
         """
         retrieve all synced data for an export
+
+        :param int limit: Max number of records to return from export
         """
 
         if self.job['job_type'] == 'imports':
             raise Exception('not an export')
 
         endpoint = self.job_def['uri'] + '/data'
 
-        return_data = self.get_data(endpoint=endpoint)
+        return_data = self.get_data(endpoint=endpoint, max_recs=max_recs)
 
         return return_data
 

pyeloqua/test/test_bulk_getdata.py

@@ -239,7 +239,8 @@ def test_get_export_data_call(mock_data):
     bulk.job_def = EXPORT_JOB_DEF
     mock_data.return_value = RETURN_DATA['items']
     bulk.get_export_data()
-    mock_data.assert_called_with(endpoint='/contacts/exports/1/data')
+    mock_data.assert_called_with(endpoint='/contacts/exports/1/data',
+                                 max_recs=None)
 
 
 @patch('pyeloqua.bulk.Bulk.get_data')

setup.py

@@ -2,7 +2,7 @@
 
 from setuptools import setup, find_packages
 
-__version__ = '0.5.5'
+__version__ = '0.5.6'
 
 def readme():
     """ open readme for long_description """