diff --git a/ipwhois/experimental.py b/ipwhois/experimental.py index 8f6ce230..81b4be1e 100644 --- a/ipwhois/experimental.py +++ b/ipwhois/experimental.py @@ -112,7 +112,7 @@ def get_bulk_asn_whois(addresses=None, retry_count=3, timeout=120): def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, excluded_entities=None, rate_limit_timeout=60, - socket_timeout=10, asn_timeout=240, proxy_openers=None): + socket_timeout=10, asn_timeout=240, http_clients=None): """ The function for bulk retrieving and parsing whois information for a list of IP addresses via HTTP (RDAP). This bulk lookup method uses bulk @@ -138,8 +138,8 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, connections in seconds. Defaults to 10. asn_timeout (:obj:`int`): The default timeout for bulk ASN lookups in seconds. Defaults to 240. - proxy_openers (:obj:`list` of :obj:`OpenerDirector`): Proxy openers - for single/rotating proxy support. Defaults to None. + http_clients (:obj:`list` of :obj:`httpx.Client`): httpx clients + for single/rotating proxy and fingerprint support. Defaults to None. Returns: namedtuple: @@ -209,11 +209,11 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, } asn_parsed_results = {} - if proxy_openers is None: + if http_clients is None: - proxy_openers = [None] + http_clients = [None] - proxy_openers_copy = iter(proxy_openers) + http_clients_copy = iter(http_clients) # Make sure addresses is unique unique_ip_list = list(unique_everseen(addresses)) @@ -347,19 +347,19 @@ def bulk_lookup_rdap(addresses=None, inc_raw=False, retry_count=3, depth=0, rate_tracker[rir]['count'] += 1 - # Get the next proxy opener to use, or None + # Get the next HTTP client object to use, or None try: - opener = next(proxy_openers_copy) + client = next(http_clients_copy) # Start at the beginning if all have been used except StopIteration: - proxy_openers_copy = iter(proxy_openers) - opener = next(proxy_openers_copy) + http_clients_copy = iter(http_clients) + client = next(http_clients_copy) # Instantiate the objects needed for the RDAP lookup - net = Net(ip, timeout=socket_timeout, proxy_opener=opener) + net = Net(ip, timeout=socket_timeout, http_client=client) rdap = RDAP(net) try: diff --git a/ipwhois/ipwhois.py b/ipwhois/ipwhois.py index 946c64b5..aa824cd2 100644 --- a/ipwhois/ipwhois.py +++ b/ipwhois/ipwhois.py @@ -40,14 +40,14 @@ class IPWhois: An IPv4 or IPv6 address timeout (:obj:`int`): The default timeout for socket connections in seconds. Defaults to 5. - proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for - proxy support. Defaults to None. + http_client (:obj:`httpx.Client`): HTTP client object. Proxies are here. + Defaults to None. """ - def __init__(self, address, timeout=5, proxy_opener=None): + def __init__(self, address, timeout=5, http_client=None): self.net = Net( - address=address, timeout=timeout, proxy_opener=proxy_opener + address=address, timeout=timeout, http_client=http_client ) self.ipasn = IPASN(self.net) @@ -61,7 +61,7 @@ def __init__(self, address, timeout=5, proxy_opener=None): def __repr__(self): return 'IPWhois({0}, {1}, {2})'.format( - self.address_str, str(self.timeout), repr(self.net.opener) + self.address_str, str(self.timeout), repr(self.net.http_client) ) def lookup_whois(self, inc_raw=False, retry_count=3, get_referral=False, diff --git a/ipwhois/net.py b/ipwhois/net.py index 47468d57..a772a749 100644 --- a/ipwhois/net.py +++ b/ipwhois/net.py @@ -49,22 +49,11 @@ IPv4Address, IPv6Address) -try: # pragma: no cover - from urllib.request import (OpenerDirector, - ProxyHandler, - build_opener, - Request, - URLError, - HTTPError) - from urllib.parse import urlencode -except ImportError: # pragma: no cover - from urllib2 import (OpenerDirector, - ProxyHandler, - build_opener, - Request, - URLError, - HTTPError) - from urllib import urlencode +from httpx import (Client, + HTTPStatusError, + TransportError, + InvalidURL) +from urllib.parse import urlencode log = logging.getLogger(__name__) @@ -101,15 +90,15 @@ class Net: An IPv4 or IPv6 address timeout (:obj:`int`): The default timeout for socket connections in seconds. Defaults to 5. - proxy_opener (:obj:`urllib.request.OpenerDirector`): The request for - proxy support. Defaults to None. + http_client (:obj:`httpx.client`): httpx client allows you to customize + usage of HTTP by this lib. Proxies are also configured via it. Raises: IPDefinedError: The address provided is defined (does not need to be resolved). """ - def __init__(self, address, timeout=5, proxy_opener=None): + def __init__(self, address, timeout=5, http_client=None): # IPv4Address or IPv6Address if isinstance(address, IPv4Address) or isinstance( @@ -129,15 +118,10 @@ def __init__(self, address, timeout=5, proxy_opener=None): self.dns_resolver.timeout = timeout self.dns_resolver.lifetime = timeout - # Proxy opener. - if isinstance(proxy_opener, OpenerDirector): + if not http_client: + http_client = Client() - self.opener = proxy_opener - - else: - - handler = ProxyHandler() - self.opener = build_opener(handler) + self.http_client = http_client # IP address in string format for use in queries. self.address_str = self.address.__str__() @@ -709,10 +693,10 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120, return d - except HTTPError as e: # pragma: no cover + except HTTPStatusError as e: # pragma: no cover # RIPE is producing this HTTP error rather than a JSON error. - if e.code == 429: + if e.response.status_code == 429: log.debug('HTTP query rate limit exceeded.') @@ -737,7 +721,7 @@ def get_http_json(self, url=None, retry_count=3, rate_limit_timeout=120, raise HTTPLookupError('HTTP lookup failed for {0} with error ' 'code {1}.'.format(url, str(e.code))) - except (URLError, socket.timeout, socket.error) as e: + except (TransportError,) as e: log.debug('HTTP query socket error: {0}'.format(e)) if retry_count > 0: @@ -865,22 +849,10 @@ def get_http_raw(self, url=None, retry_count=3, headers=None, # Create the connection for the HTTP query. log.debug('HTTP query for {0} at {1}'.format( self.address_str, url)) - try: - # Py 2 inspection alert bypassed by using kwargs dict. - conn = Request(url=url, data=enc_form_data, headers=headers, - **{'method': request_type}) - except TypeError: # pragma: no cover - conn = Request(url=url, data=enc_form_data, headers=headers) - data = self.opener.open(conn, timeout=self.timeout) - - try: - d = data.readall().decode('ascii', 'ignore') - except AttributeError: # pragma: no cover - d = data.read().decode('ascii', 'ignore') - - return str(d) + return self.http_client.request(url=url, data=enc_form_data, headers=headers, + **{'method': request_type}).text - except (URLError, socket.timeout, socket.error) as e: + except (InvalidURL, TransportError) as e: log.debug('HTTP query socket error: {0}'.format(e)) if retry_count > 0: diff --git a/ipwhois/scripts/ipwhois_cli.py b/ipwhois/scripts/ipwhois_cli.py index c36b7919..ce066baf 100644 --- a/ipwhois/scripts/ipwhois_cli.py +++ b/ipwhois/scripts/ipwhois_cli.py @@ -31,12 +31,7 @@ from ipwhois.hr import (HR_ASN, HR_RDAP, HR_RDAP_COMMON, HR_WHOIS, HR_WHOIS_NIR) -try: # pragma: no cover - from urllib.request import (ProxyHandler, - build_opener) -except ImportError: # pragma: no cover - from urllib2 import (ProxyHandler, - build_opener) +from httpx import Client # CLI ANSI rendering ANSI = { @@ -348,18 +343,15 @@ class IPWhoisCLI: An IPv4 or IPv6 address timeout (:obj:`int`): The default timeout for socket connections in seconds. Defaults to 5. - proxy_http (:obj:`urllib.request.OpenerDirector`): The request for - proxy HTTP support or None. - proxy_https (:obj:`urllib.request.OpenerDirector`): The request for - proxy HTTPS support or None. + http_client (:obj:`httpx.Client`): The httpx.Client objects. + Proxies and not only are here. """ def __init__( self, addr, timeout, - proxy_http, - proxy_https + http_client ): self.addr = addr @@ -368,29 +360,28 @@ def __init__( handler_dict = None if proxy_http is not None: - handler_dict = {'http': proxy_http} + handler_dict = {'http://*': proxy_http} if proxy_https is not None: if handler_dict is None: - handler_dict = {'https': proxy_https} + handler_dict = {'https://*': proxy_https} else: - handler_dict['https'] = proxy_https + handler_dict['https://*'] = proxy_https if handler_dict is None: - self.opener = None + self.http_client = None else: - handler = ProxyHandler(handler_dict) - self.opener = build_opener(handler) + self.http_client = Client(proxies=handler_dict) self.obj = IPWhois(address=self.addr, timeout=self.timeout, - proxy_opener=self.opener) + http_client=self.http_client) def generate_output_header(self, query_type='RDAP'): """ diff --git a/ipwhois/tests/online/test_experimental.py b/ipwhois/tests/online/test_experimental.py index 6ea9b473..bd49d838 100644 --- a/ipwhois/tests/online/test_experimental.py +++ b/ipwhois/tests/online/test_experimental.py @@ -2,6 +2,7 @@ from ipwhois.tests import TestCommon from ipwhois.exceptions import (ASNLookupError) from ipwhois.experimental import (get_bulk_asn_whois, bulk_lookup_rdap) +from httpx import Client LOG_FORMAT = ('[%(asctime)s] [%(levelname)s] [%(filename)s:%(lineno)s] ' '[%(funcName)s()] %(message)s') @@ -39,18 +40,9 @@ def test_get_bulk_asn_whois(self): def test_bulk_lookup_rdap(self): - try: - from urllib.request import (OpenerDirector, - ProxyHandler, - build_opener) - except ImportError: - from urllib2 import (OpenerDirector, - ProxyHandler, - build_opener) + from httpx import Client - handler = ProxyHandler() - opener = build_opener(handler) - bulk_lookup_rdap(addresses=['74.125.225.229'], proxy_openers=[opener]) + bulk_lookup_rdap(addresses=['74.125.225.229'], http_client=Client()) ips = [ '74.125.225.229', # ARIN diff --git a/ipwhois/tests/online/test_ipwhois.py b/ipwhois/tests/online/test_ipwhois.py index 3627e9d7..bd821b57 100644 --- a/ipwhois/tests/online/test_ipwhois.py +++ b/ipwhois/tests/online/test_ipwhois.py @@ -125,10 +125,7 @@ def test_lookup_whois(self): break def test_lookup_rdap(self): - try: - from urllib.request import ProxyHandler, build_opener - except ImportError: - from urllib2 import ProxyHandler, build_opener + from httpx import Client ips = [ '74.125.225.229', # ARIN @@ -169,8 +166,7 @@ def test_lookup_rdap(self): except Exception as e: self.fail('Unexpected exception raised: {0}'.format(e)) - handler = ProxyHandler({'http': 'http://0.0.0.0:80/'}) - opener = build_opener(handler) + http_client = Client(proxies={'http://*': 'http://0.0.0.0:80/'}) result = IPWhois(address='74.125.225.229', timeout=0, - proxy_opener=opener) + http_client=http_client) self.assertRaises(ASNRegistryError, result.lookup_rdap) diff --git a/ipwhois/tests/test_net.py b/ipwhois/tests/test_net.py index aedf6ac2..a6477a36 100644 --- a/ipwhois/tests/test_net.py +++ b/ipwhois/tests/test_net.py @@ -38,20 +38,12 @@ def test_timeout(self): result = Net('74.125.225.229') self.assertIsInstance(result.timeout, int) - def test_proxy_opener(self): - try: - from urllib.request import (OpenerDirector, - ProxyHandler, - build_opener) - except ImportError: - from urllib2 import (OpenerDirector, - ProxyHandler, - build_opener) + def test_http_client(self): + from httpx import Client result = Net('74.125.225.229') - self.assertIsInstance(result.opener, OpenerDirector) + self.assertIsInstance(result.http_client, Client) - handler = ProxyHandler() - opener = build_opener(handler) - result = Net(address='74.125.225.229', proxy_opener=opener) - self.assertIsInstance(result.opener, OpenerDirector) + client = Client() + result = Net(address='74.125.225.229', http_client=client) + self.assertIsInstance(result.http_client, Client) diff --git a/setup.py b/setup.py index ef22a097..3bf47dbb 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ PACKAGE_DATA = {'ipwhois': ['data/*.xml', 'data/*.csv']} -INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"'] +INSTALL_REQUIRES = ['dnspython<=2.0.0', 'ipaddr==2.2.0;python_version<"3.3"', 'httpx'] setup( name=NAME,