0.6 - upgrade search result list to a custom class to enable extracti…

…on of more results & total number of results

duedil/api.py

@@ -19,9 +19,9 @@
 from __future__ import unicode_literals
 
 
-from .search.lite import CompanySearchResult as LiteCompanySearchResult
-from .search.pro import CompanySearchResult as ProCompanySearchResult, DirectorSearchResult
-from .search.international import CompanySearchResult as InternationalCompanySearchResult
+from .search.lite import CompanySearchResult as LiteCompanySearchResult, LiteSearchResourceList
+from .search.pro import CompanySearchResult as ProCompanySearchResult, DirectorSearchResult, ProSearchResourceList
+from .search.international import CompanySearchResult as InternationalCompanySearchResult, InternationalSearchResourceList
 
 from .cache import configure_cache, dp_region as cache_region
 
@@ -102,6 +102,7 @@ def post_request_hook(self, response):
         you cannot affect further processing of the response'''
         pass
 
+# - _get should probably be split a bit more to allow full urls to be called
     @cache_region.cache_on_arguments()
     @retry(retry_on_exception=retry_throttling, wait_exponential_multiplier=1000, wait_exponential_max=10000)
     def _get(self, endpoint, data=None):
@@ -141,7 +142,8 @@ def _get(self, endpoint, data=None):
     def _search(self, endpoint, result_klass, *args, **kwargs):
         query_params = self._build_search_string(*args, **kwargs)
         results = self._get(endpoint, data=query_params)
-        return [result_klass(self, **r) for r in results.get('response',{}).get('data', {})]
+        # return [result_klass(self, **r) for r in results.get('response',{}).get('data', {})]
+        return self.search_list_class(results, result_klass, self)
 
     def _build_search_string(self, *args, **kwargs):
         data = {}
@@ -157,6 +159,7 @@ def search(self, query):
 
 class LiteClient(Client):
     api_type = 'lite'
+    search_list_class = LiteSearchResourceList
 
     def search(self, query):
         #  this will need to be alter in all likely hood to do some validation
@@ -165,6 +168,7 @@ def search(self, query):
 
 class ProClient(Client):
     api_type = 'pro'
+    search_list_class = ProSearchResourceList
 
     def _build_search_string(self, term_filters, range_filters,
                              order_by=None, limit=None, offset=None,
@@ -283,6 +287,7 @@ def search(self, order_by=None, limit=None, offset=None, **kwargs):
 
 class InternationalClient(Client):
     api_type = 'international'
+    search_list_class = InternationalSearchResourceList
 
     def search(self, country_code, query):
         endpoint = '{}/search'.format(country_code)

duedil/resources/__init__.py

@@ -20,7 +20,7 @@
 
 import sys
 import six
-from collections import MutableMapping
+from collections import Mapping
 from abc import ABCMeta
 
 from ..api import LiteClient, ProClient  # , InternationalClient
@@ -30,7 +30,7 @@ class ReadOnlyException(Exception):
     pass
 
 
-class Resource(MutableMapping):
+class Resource(Mapping):
     attribute_names = None
     locale = 'uk'
     id = None
@@ -101,12 +101,6 @@ def __len__(self):
     def __getitem__(self, key):
         return self.__getattr__(key)
 
-    def __setitem__(self, key, item):
-        raise ReadOnlyException('This is a read-only API so you cannot set attributes')
-
-    def __delitem__(self, key):
-        raise ReadOnlyException('This is a read-only API so you cannot delete attributes')
-
     def __iter__(self):
         self.load()
         return iter(self._result)

duedil/search/__init__.py

@@ -1,5 +1,8 @@
 
 from importlib import import_module
+from collections import Sequence
+
+
 
 class SearchResource(object):
     attribute_names = None
@@ -63,3 +66,51 @@ def __eq__(self, other):
         if hasattr(other, 'id'):
             return self.id == other.id
         return self.id == other
+
+
+
+class SearchResouceList(Sequence):
+
+    def __init__(self, results, result_klass, client):
+        self._result_list = []
+        self.result_klass = result_klass
+        self.client = client
+        self.result_list = results
+
+    def __len__(self):
+        return len(self.result_list)
+
+    def __getitem__(self, key):
+        return self.result_list[key]
+
+    def __iter__(self):
+        return iter(self.result_list)
+
+    def __contains__(self, result):
+        return result in self.result_list
+
+    def __eq__(self, other):
+        rlist = self.result_list == other.result_list
+        client = self.client == other.client
+        rclass = self.result_klass == other.result_klass
+        return rlist and client and rclass
+
+    def __add__(self, other):
+        if self.client.api_key == other.client.api_key:
+            return self.result_list + other.result_list
+        raise TypeError('Cannot join results from 2 different applications (api keys)')
+
+    def __radd__(self, other):
+        return self.__add__(other)
+
+    @property
+    def result_list(self):
+        return self._result_list
+
+    @result_list.setter
+    def result_list(self, value):
+        temp = [self.result_klass(self.client, **r) for r in value.get('response',{}).get('data', {})]
+        if not self._result_list:
+            self._result_list = temp
+        else:
+            self._result_list.extend(temp)

duedil/search/international/__init__.py

@@ -1 +1,6 @@
 from .company import CompanySearchResult
+from .. import SearchResouceList
+
+
+class InternationalSearchResourceList(SearchResouceList):
+    pass

duedil/search/lite/__init__.py

@@ -1 +1,6 @@
 from .company import CompanySearchResult
+from .. import SearchResouceList
+
+
+class LiteSearchResourceList(SearchResouceList):
+    pass

duedil/search/pro/__init__.py

@@ -1,2 +1,98 @@
 from .company import CompanySearchResult
 from .director import DirectorSearchResult
+from .. import SearchResouceList
+
+from collections import Sequence
+from urllib import urlencode
+import urlparse
+
+class ProSearchResourceList(SearchResouceList):
+
+    def __init__(self, results, result_klass, client):
+        super(ProSearchResourceList, self).__init__(results, result_klass, client)
+        self._next_url = None
+        # look at the property below
+        self.next_url = results
+        page = results.get('response', {}).get('pagination')
+        self._length = page.get('total', len(self.result_list))
+
+    @property
+    def next_url(self):
+        return self._next_url
+
+    @next_url.setter
+    def next_url(self, value):
+        page = value.get('response', {}).get('pagination')
+        if page:
+            self._next_url = page.get('next_url')
+        else:
+            self._next_url = ''
+
+
+    def next(self):
+        # should get the next set of results
+        # update the internal list
+        if not self.fetched_all_results():
+            next_set = self.client.get(*self.parse_next_url())
+            self.result_list = next_set
+            # update the next_url
+            self.next_url = next_set
+        else:
+            raise StopIteration
+
+    def __len__(self):
+        return self._length
+
+    def __getitem__(self, key):
+        if type(key) is int:
+            if key < 0:
+                # key should be negative so +- equals -
+                key = self._length + key
+            if key >= self._length:
+                raise IndexError()
+            try:
+                return self.result_list[key]
+            except IndexError:
+                # key is not in current cut of data
+                # therefore loop with .next until result?
+                self._update_next_url(1, key)
+                return self.client.get(*self.parse_next_url())
+        elif type(key) is slice:
+            raise NotImplementedError("Results don't support slicing at this time")
+        else:
+            raise TypeError()
+
+    def __iter__(self):
+        for result in self.result_list:
+            yield result
+            # get the last one and extend the list
+            if self.result_list[-1] is result:
+                self.next()
+
+
+    def __contains__(self, result):
+        while not self.fetched_all_results():
+            if result in self.result_list:
+                return True
+            # this could be hugely expensive, so get as many as possible. Duedil only allow upto a limit of 100
+            self._update_next_url(limit=100)
+            self.next()
+        return False
+
+    def fetched_all_results(self):
+        return len(self.result_list) >= len(self)
+
+    def _update_next_url(self, limit=None, offset=None):
+        scheme, netloc, path, query_string, frag = urlparse.urlsplit(self._next_url)
+        query_params = urlparse.parsed_qs(query_string)
+        if limit:
+            query_params['limit'] = [limit]
+        if offset:
+            query_params['offset'] = [offset]
+        self._next_url = urlparse.urlunsplit((scheme, netloc, path, urlencode(query_params, doseq=True), frag))
+
+    def parse_next_url(self):
+        parsed_url = urlparse.urlunsplit(self._next_url)
+        path = parsed_url.path.rsplit('/', 1)[-1] # grab the last part of the path
+        query_params = dict(urlparse.parsed_qsl(parsed_url.query))
+        return path, query_params

setup.py

@@ -25,7 +25,7 @@ def run_tests(self):
         sys.exit(errno)
 
 
-version = '0.5.6'
+version = '0.6.0'
 
 setup(name='duedil',
       version=version,

tests/test_client.py

@@ -29,6 +29,7 @@
 from duedil.resources.lite import Company as LiteCompany
 from duedil.search.pro import CompanySearchResult as ProCompanySearchResult, DirectorSearchResult
 from duedil.search.lite import CompanySearchResult as LiteCompanySearchResult
+from duedil.search.international import CompanySearchResult as InternationalCompanySearchResult, InternationalSearchResourceList
 
 API_KEY = '12345'
 
@@ -214,7 +215,7 @@ def test_search_company(self, m):
         }
         url = 'http://duedil.io/v3/companies.json'
         m.register_uri('GET', url,
-                       json={'response': {'data': [result]}})
+                       json={'response': {'data': [result], 'pagination':{'total':1}}})
         companies = self.client.search_company()
         self.assertEqual(len(companies), 1)
         self.assertIn('api_key=12345', m._adapter.last_request.query)
@@ -231,7 +232,7 @@ def test_search_director(self, m):
         }
         url = 'http://duedil.io/v3/directors.json'
         m.register_uri('GET', url,
-                       json={'response': {'data': [result]}})
+                       json={'response': {'data': [result], 'pagination':{'total':1}}})
         directors = self.client.search_director()
         self.assertEqual(len(directors), 1)
         self.assertIn('api_key=12345', m._adapter.last_request.query)
@@ -248,7 +249,7 @@ def test_search(self, m):
         }
         url = 'http://duedil.io/v3/directors.json'
         m.register_uri('GET', url,
-                       json={'response': {'data': [result]}})
+                       json={'response': {'data': [result], 'pagination':{'total':1}}})
         result = {
             'locale': 'uk',
             'url': 'http://duedil.io/v3/uk/companies/06999618.json',
@@ -257,7 +258,7 @@ def test_search(self, m):
         }
         url = 'http://duedil.io/v3/companies.json'
         m.register_uri('GET', url,
-                       json={'response': {'data': [result]}})
+                       json={'response': {'data': [result], 'pagination':{'total':1}}})
         results = self.client.search()
         self.assertEqual(len(results), 2)
         self.assertIn('api_key=12345', m._adapter.last_request.query)
@@ -272,7 +273,9 @@ class SearchQueryTestCase(unittest.TestCase):
     @requests_mock.mock()
     def test_search_filter(self, m):
         m.register_uri('GET', self.url,
-                       json={})
+                       json={
+                       'response':{'pagination':{'total':0}}
+                       })
         self.client.search_company(name='DueDil')
         self.assertEqual(
             json.loads(m._adapter.last_request.qs['filters'][0]),
@@ -281,7 +284,9 @@ def test_search_filter(self, m):
     @requests_mock.mock()
     def test_search_range(self, m):
         m.register_uri('GET', self.url,
-                       json={})
+                       json={
+                        'response':{'pagination':{'total':0}}
+                       })
         self.client.search_company(employee_count=(1, 100,))
         self.assertEqual(
             json.loads(m._adapter.last_request.qs['filters'][0]),
@@ -290,14 +295,18 @@ def test_search_range(self, m):
     @requests_mock.mock()
     def test_search_non_filter_or_range(self, m):
         m.register_uri('GET', self.url,
-                       json={})
+                       json={
+                       'response':{'pagination':{'total':0}}
+                       })
         with self.assertRaises(TypeError):
             self.client.search_company(non_filter="Not Implemented")
 
     @requests_mock.mock()
     def test_search_order(self, m):
         m.register_uri('GET', self.url,
-                       json={})
+                       json={
+                       'response':{'pagination':{'total':0}}
+                       })
         self.client.search_company(order_by={'field': 'name',
                                              'direction': 'asc'})
         self.assertEqual(
@@ -307,14 +316,18 @@ def test_search_order(self, m):
     @requests_mock.mock()
     def test_search_limit(self, m):
         m.register_uri('GET', self.url,
-                       json={})
+                       json={
+                       'response':{'pagination':{'total':0}}
+                       })
         self.client.search_company(limit=100)
         self.assertEqual(m._adapter.last_request.qs['limit'][0], '100')
 
     @requests_mock.mock()
     def test_search_offset(self, m):
         m.register_uri('GET', self.url,
-                       json={})
+                       json={
+                       'response':{'pagination':{'total':0}}
+                       })
         self.client.search_company(offset=50)
         self.assertEqual(m._adapter.last_request.qs['offset'][0], '50')
 
@@ -338,7 +351,7 @@ def test_report(self, m):
     @requests_mock.mock()
     def test_search(self, m):
         m.register_uri('GET', 'http://api.duedil.com/international/uk/search?q=Acme', json={})
-        self.assertEqual(self.client.search('uk', 'Acme'), [])
+        self.assertEqual(self.client.search('uk', 'Acme'), InternationalSearchResourceList({}, InternationalCompanySearchResult, self.client))
 
 
 def test_suite():