From a0b7d50d0804d2486a24246e505c4b4910fdee71 Mon Sep 17 00:00:00 2001 From: Kevin McDonald Date: Wed, 13 Mar 2013 10:46:09 -0500 Subject: [PATCH 1/2] Adds Result Generator for API calls. This allows for iterating over a set of results in chunks so you don't have to transfer and then hold the entire dataset in memory. >>> cci_iter = client['Account'].getVirtualGuests(iter=True, chunk=100, ...) >>> for cci in cci_iter: >>> print cci['fqdn'] Still needs tests. --- SoftLayer/API.py | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/SoftLayer/API.py b/SoftLayer/API.py index ac1aaa4f0..1617f83f8 100644 --- a/SoftLayer/API.py +++ b/SoftLayer/API.py @@ -263,6 +263,33 @@ def __call__(self, service, method, *args, **kwargs): http_headers=http_headers, timeout=self.timeout, verbose=self.verbose) + def iter_call(self, service, method, + chunk=100, limit=None, offset=0, **kwargs): + if chunk <= 0: + raise AttributeError("Chunk size should be greater than zero.") + + if limit: + chunk = min(chunk, limit) + + while True: + results = self(service, method, + offset=offset, limit=chunk, **kwargs) + if not results: + break + + if not isinstance(results, list): + yield results + break + + for item in results: + yield item + offset += 1 + if offset > limit: + break + + if offset >= limit: + break + def __format_object_mask(self, objectmask, service): """ Format new and old style object masks into proper headers. @@ -304,12 +331,16 @@ def __getattr__(self, name): if name in ["__name__", "__bases__"]: raise AttributeError("'Obj' object has no attribute '%s'" % name) - def call_handler(*args, **kwargs): + def call_handler(iter=False, *args, **kwargs): if self._service_name is None: raise SoftLayerError( "Service is not set on Client instance.") kwargs['headers'] = self._headers - return self(self._service_name, name, *args, **kwargs) + if iter: + return self.iter_call(self._service_name, name, + *args, **kwargs) + else: + return self(self._service_name, name, *args, **kwargs) return call_handler def __repr__(self): From 81b416a146c450e922ba006a5d8b4cdff923cd77 Mon Sep 17 00:00:00 2001 From: Kevin McDonald Date: Wed, 13 Mar 2013 14:51:51 -0500 Subject: [PATCH 2/2] Finishes first implementation of Generator Example usage below. This will request 10 VMs at a time from the API until 30 total starting after the first 10. >>> for i, vm in enumerate(c['SoftLayer_Account'].getVirtualGuests(iter=True, chunk=10, limit=30, offset=10)): >>> print i, vm['fullyQualifiedDomainName'] --- SoftLayer/API.py | 43 ++++++++++++------ SoftLayer/tests/API/client_tests.py | 68 ++++++++++++++++++++++++++++- 2 files changed, 96 insertions(+), 15 deletions(-) diff --git a/SoftLayer/API.py b/SoftLayer/API.py index 1617f83f8..ee439b4bf 100644 --- a/SoftLayer/API.py +++ b/SoftLayer/API.py @@ -209,7 +209,7 @@ def __getitem__(self, name): name = self._prefix + name return Service(self, name) - def __call__(self, service, method, *args, **kwargs): + def __call__(self, *args, **kwargs): """ Perform a SoftLayer API call. :param service: the name of the SoftLayer API service @@ -218,6 +218,12 @@ def __call__(self, service, method, *args, **kwargs): :param dict **kwargs: response-level arguments (limit, offset, etc.) """ + if kwargs.get('iter'): + return self.iter_call(*args, **kwargs) + else: + return self.call(*args, **kwargs) + + def call(self, service, method, *args, **kwargs): objectid = kwargs.get('id') objectmask = kwargs.get('mask') objectfilter = kwargs.get('filter') @@ -264,30 +270,43 @@ def __call__(self, service, method, *args, **kwargs): verbose=self.verbose) def iter_call(self, service, method, - chunk=100, limit=None, offset=0, **kwargs): + chunk=100, limit=None, offset=0, *args, **kwargs): if chunk <= 0: raise AttributeError("Chunk size should be greater than zero.") if limit: chunk = min(chunk, limit) + result_count = 0 while True: - results = self(service, method, - offset=offset, limit=chunk, **kwargs) + if limit: + # We've reached the end of the results + if result_count >= limit: + break + + # Don't over-fetch past the given limit + if chunk + result_count > limit: + chunk = limit - result_count + results = self.call(service, method, + offset=offset, limit=chunk, *args, **kwargs) + + # It looks like we ran out results if not results: break + # Apparently this method doesn't return a list. + # Why are you even iterating over this? if not isinstance(results, list): yield results break for item in results: yield item - offset += 1 - if offset > limit: - break + result_count += 1 - if offset >= limit: + offset += chunk + + if len(results) < chunk: break def __format_object_mask(self, objectmask, service): @@ -331,16 +350,12 @@ def __getattr__(self, name): if name in ["__name__", "__bases__"]: raise AttributeError("'Obj' object has no attribute '%s'" % name) - def call_handler(iter=False, *args, **kwargs): + def call_handler(*args, **kwargs): if self._service_name is None: raise SoftLayerError( "Service is not set on Client instance.") kwargs['headers'] = self._headers - if iter: - return self.iter_call(self._service_name, name, - *args, **kwargs) - else: - return self(self._service_name, name, *args, **kwargs) + return self(self._service_name, name, *args, **kwargs) return call_handler def __repr__(self): diff --git a/SoftLayer/tests/API/client_tests.py b/SoftLayer/tests/API/client_tests.py index f4b02f4b3..5ef9ea6ee 100644 --- a/SoftLayer/tests/API/client_tests.py +++ b/SoftLayer/tests/API/client_tests.py @@ -3,7 +3,7 @@ except ImportError: import unittest # NOQA -from mock import patch, MagicMock +from mock import patch, MagicMock, call import SoftLayer import SoftLayer.API @@ -262,3 +262,69 @@ def test_mask_call_invalid_mask(self, make_api_call): self.assertIn('Malformed Mask', str(e)) else: self.fail('No exception raised') + + @patch('SoftLayer.API.Client.call') + def test_iterate(self, _call): + # chunk=100, no limit + _call.side_effect = [range(100), range(100, 125)] + result = list(self.client['SERVICE'].METHOD(iter=True)) + + self.assertEquals(range(125), result) + _call.assert_has_calls([ + call('SoftLayer_SERVICE', 'METHOD', + limit=100, iter=True, offset=0), + call('SoftLayer_SERVICE', 'METHOD', + limit=100, iter=True, offset=100), + ]) + _call.reset_mock() + + # chunk=100, no limit. Requires one extra request. + _call.side_effect = [range(100), range(100, 200), []] + result = list(self.client['SERVICE'].METHOD(iter=True)) + self.assertEquals(range(200), result) + _call.assert_has_calls([ + call('SoftLayer_SERVICE', 'METHOD', + limit=100, iter=True, offset=0), + call('SoftLayer_SERVICE', 'METHOD', + limit=100, iter=True, offset=100), + call('SoftLayer_SERVICE', 'METHOD', + limit=100, iter=True, offset=200), + ]) + _call.reset_mock() + + # chunk=25, limit=30 + _call.side_effect = [range(0, 25), range(25, 30)] + result = list( + self.client['SERVICE'].METHOD(iter=True, limit=30, chunk=25)) + self.assertEquals(range(30), result) + _call.assert_has_calls([ + call('SoftLayer_SERVICE', 'METHOD', iter=True, limit=25, offset=0), + call('SoftLayer_SERVICE', 'METHOD', iter=True, limit=5, offset=25), + ]) + _call.reset_mock() + + # A non-list was returned + _call.side_effect = ["test"] + result = list(self.client['SERVICE'].METHOD(iter=True)) + self.assertEquals(["test"], result) + _call.assert_has_calls([ + call('SoftLayer_SERVICE', 'METHOD', + iter=True, limit=100, offset=0), + ]) + _call.reset_mock() + + # chunk=25, limit=30, offset=12 + _call.side_effect = [range(0, 25), range(25, 30)] + result = list(self.client['SERVICE'].METHOD( + iter=True, limit=30, chunk=25, offset=12)) + self.assertEquals(range(30), result) + _call.assert_has_calls([ + call('SoftLayer_SERVICE', 'METHOD', + iter=True, limit=25, offset=12), + call('SoftLayer_SERVICE', 'METHOD', iter=True, limit=5, offset=37), + ]) + + # Chunk size of 0 is invalid + self.assertRaises( + AttributeError, + lambda: list(self.client['SERVICE'].METHOD(iter=True, chunk=0)))