From 46124601d92908456d16e49bbfe0814bd51123ad Mon Sep 17 00:00:00 2001 From: Jason Madden Date: Fri, 3 May 2019 07:39:10 -0500 Subject: [PATCH] Make dnspython optional for testing. --- src/gevent/resolver/_addresses.py | 163 +++++++++++++++++++++++++++ src/gevent/resolver/_hostsfile.py | 145 ++++++++++++++++++++++++ src/gevent/resolver/dnspython.py | 135 +--------------------- src/gevent/testing/modules.py | 17 ++- src/gevent/tests/test___config.py | 13 ++- src/gevent/tests/test__all__.py | 16 ++- src/gevent/tests/test__socket_dns.py | 2 +- 7 files changed, 344 insertions(+), 147 deletions(-) create mode 100644 src/gevent/resolver/_addresses.py create mode 100644 src/gevent/resolver/_hostsfile.py diff --git a/src/gevent/resolver/_addresses.py b/src/gevent/resolver/_addresses.py new file mode 100644 index 000000000..6287d379c --- /dev/null +++ b/src/gevent/resolver/_addresses.py @@ -0,0 +1,163 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 gevent contributors. See LICENSE for details. +# +# Portions of this code taken from dnspython +# https://github.com/rthalley/dnspython +# +# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license + +# Copyright (C) 2003-2017 Nominum, Inc. +# +# Permission to use, copy, modify, and distribute this software and its +# documentation for any purpose with or without fee is hereby granted, +# provided that the above copyright notice and this permission notice +# appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +""" +Private support for parsing textual addresses. + +""" +from __future__ import absolute_import, division, print_function + +import binascii +import re + +from gevent.resolver import hostname_types + + +class AddressSyntaxError(ValueError): + pass + + +def _ipv4_inet_aton(text): + """ + Convert an IPv4 address in text form to binary struct. + + *text*, a ``text``, the IPv4 address in textual form. + + Returns a ``binary``. + """ + + if not isinstance(text, bytes): + text = text.encode() + parts = text.split(b'.') + if len(parts) != 4: + raise AddressSyntaxError(text) + for part in parts: + if not part.isdigit(): + raise AddressSyntaxError + if len(part) > 1 and part[0] == '0': + # No leading zeros + raise AddressSyntaxError(text) + try: + ints = [int(part) for part in parts] + return struct.pack('BBBB', *ints) + except: + raise AddressSyntaxError(text) + + +def _ipv6_inet_aton(text, + _v4_ending=re.compile(br'(.*):(\d+\.\d+\.\d+\.\d+)$'), + _colon_colon_start=re.compile(br'::.*'), + _colon_colon_end=re.compile(br'.*::$')): + """ + Convert an IPv6 address in text form to binary form. + + *text*, a ``text``, the IPv6 address in textual form. + + Returns a ``binary``. + """ + # pylint:disable=too-many-branches + + # + # Our aim here is not something fast; we just want something that works. + # + if not isinstance(text, bytes): + text = text.encode() + + if text == b'::': + text = b'0::' + # + # Get rid of the icky dot-quad syntax if we have it. + # + m = _v4_ending.match(text) + if not m is None: + b = bytearray(_ipv4_inet_aton(m.group(2))) + text = (u"{}:{:02x}{:02x}:{:02x}{:02x}".format(m.group(1).decode(), + b[0], b[1], b[2], + b[3])).encode() + # + # Try to turn '::' into ':'; if no match try to + # turn '::' into ':' + # + m = _colon_colon_start.match(text) + if not m is None: + text = text[1:] + else: + m = _colon_colon_end.match(text) + if not m is None: + text = text[:-1] + # + # Now canonicalize into 8 chunks of 4 hex digits each + # + chunks = text.split(b':') + l = len(chunks) + if l > 8: + raise SyntaxError + seen_empty = False + canonical = [] + for c in chunks: + if c == b'': + if seen_empty: + raise AddressSyntaxError(text) + seen_empty = True + for _ in range(0, 8 - l + 1): + canonical.append(b'0000') + else: + lc = len(c) + if lc > 4: + raise AddressSyntaxError(text) + if lc != 4: + c = (b'0' * (4 - lc)) + c + canonical.append(c) + if l < 8 and not seen_empty: + raise AddressSyntaxError(text) + text = b''.join(canonical) + + # + # Finally we can go to binary. + # + try: + return binascii.unhexlify(text) + except (binascii.Error, TypeError): + raise AddressSyntaxError(text) + + +def _is_addr(host, parse=_ipv4_inet_aton): + if not host: + return False + assert isinstance(host, hostname_types), repr(host) + try: + parse(host) + except AddressSyntaxError: + return False + else: + return True + +# Return True if host is a valid IPv4 address +is_ipv4_addr = _is_addr + + +def is_ipv6_addr(host): + # Return True if host is a valid IPv6 address + if host: + s = '%' if isinstance(host, str) else b'%' + host = host.split(s, 1)[0] + return _is_addr(host, _ipv6_inet_aton) diff --git a/src/gevent/resolver/_hostsfile.py b/src/gevent/resolver/_hostsfile.py new file mode 100644 index 000000000..9f92a8592 --- /dev/null +++ b/src/gevent/resolver/_hostsfile.py @@ -0,0 +1,145 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2019 gevent contributors. See LICENSE for details. +# +# Portions of this code taken from dnspython +# https://github.com/rthalley/dnspython +# +# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license + +# Copyright (C) 2003-2017 Nominum, Inc. +# +# Permission to use, copy, modify, and distribute this software and its +# documentation for any purpose with or without fee is hereby granted, +# provided that the above copyright notice and this permission notice +# appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +""" +Private support for parsing /etc/hosts. + +""" +from __future__ import absolute_import, division, print_function + +import sys +import os +import re + +from gevent.resolver._addresses import is_ipv4_addr +from gevent.resolver._addresses import is_ipv6_addr + +from gevent._compat import iteritems + + +class HostsFile(object): + """ + A class to read the contents of a hosts file (/etc/hosts). + """ + + LINES_RE = re.compile(r""" + \s* # Leading space + ([^\r\n#]+?) # The actual match, non-greedy so as not to include trailing space + \s* # Trailing space + (?:[#][^\r\n]+)? # Comments + (?:$|[\r\n]+) # EOF or newline + """, re.VERBOSE) + + def __init__(self, fname=None): + self.v4 = {} # name -> ipv4 + self.v6 = {} # name -> ipv6 + self.aliases = {} # name -> canonical_name + self.reverse = {} # ip addr -> some name + if fname is None: + if os.name == 'posix': + fname = '/etc/hosts' + elif os.name == 'nt': # pragma: no cover + fname = os.path.expandvars( + r'%SystemRoot%\system32\drivers\etc\hosts') + self.fname = fname + assert self.fname + self._last_load = 0 + + + def _readlines(self): + # Read the contents of the hosts file. + # + # Return list of lines, comment lines and empty lines are + # excluded. Note that this performs disk I/O so can be + # blocking. + with open(self.fname, 'rb') as fp: + fdata = fp.read() + + + # XXX: Using default decoding. Is that correct? + udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata + + return self.LINES_RE.findall(udata) + + def load(self): # pylint:disable=too-many-locals + # Load hosts file + + # This will (re)load the data from the hosts + # file if it has changed. + + try: + load_time = os.stat(self.fname).st_mtime + needs_load = load_time > self._last_load + except (IOError, OSError): + from gevent import get_hub + get_hub().handle_error(self, *sys.exc_info()) + needs_load = False + + if not needs_load: + return + + v4 = {} + v6 = {} + aliases = {} + reverse = {} + + for line in self._readlines(): + parts = line.split() + if len(parts) < 2: + continue + ip = parts.pop(0) + if is_ipv4_addr(ip): + ipmap = v4 + elif is_ipv6_addr(ip): + if ip.startswith('fe80'): + # Do not use link-local addresses, OSX stores these here + continue + ipmap = v6 + else: + continue + cname = parts.pop(0).lower() + ipmap[cname] = ip + for alias in parts: + alias = alias.lower() + ipmap[alias] = ip + aliases[alias] = cname + + # XXX: This is wrong for ipv6 + if ipmap is v4: + ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa' + else: + ptr = ip + '.ip6.arpa.' + if ptr not in reverse: + reverse[ptr] = cname + + self._last_load = load_time + self.v4 = v4 + self.v6 = v6 + self.aliases = aliases + self.reverse = reverse + + def iter_all_host_addr_pairs(self): + self.load() + for name, addr in iteritems(self.v4): + yield name, addr + for name, addr in iteritems(self.v6): + yield name, addr diff --git a/src/gevent/resolver/dnspython.py b/src/gevent/resolver/dnspython.py index 83281a41a..dfca656b5 100644 --- a/src/gevent/resolver/dnspython.py +++ b/src/gevent/resolver/dnspython.py @@ -60,9 +60,6 @@ # THE SOFTWARE. from __future__ import absolute_import, print_function, division -import os - -import re import sys import time @@ -78,6 +75,8 @@ from gevent.resolver import AbstractResolver from gevent.resolver import hostname_types +from gevent.resolver._hostsfile import HostsFile +from gevent.resolver._addresses import is_ipv6_addr from gevent._compat import string_types from gevent._compat import iteritems @@ -172,135 +171,7 @@ def _getaddrinfo(host=None, service=None, family=AF_UNSPEC, socktype=0, HOSTS_TTL = 300.0 -def _is_addr(host, parse=dns.ipv4.inet_aton): - if not host: - return False - assert isinstance(host, hostname_types), repr(host) - try: - parse(host) - except dns.exception.SyntaxError: - return False - else: - return True - -# Return True if host is a valid IPv4 address -_is_ipv4_addr = _is_addr - - -def _is_ipv6_addr(host): - # Return True if host is a valid IPv6 address - if host: - s = '%' if isinstance(host, str) else b'%' - host = host.split(s, 1)[0] - return _is_addr(host, dns.ipv6.inet_aton) - -class HostsFile(object): - """ - A class to read the contents of a hosts file (/etc/hosts). - """ - - LINES_RE = re.compile(r""" - \s* # Leading space - ([^\r\n#]+?) # The actual match, non-greedy so as not to include trailing space - \s* # Trailing space - (?:[#][^\r\n]+)? # Comments - (?:$|[\r\n]+) # EOF or newline - """, re.VERBOSE) - - def __init__(self, fname=None): - self.v4 = {} # name -> ipv4 - self.v6 = {} # name -> ipv6 - self.aliases = {} # name -> canonical_name - self.reverse = {} # ip addr -> some name - if fname is None: - if os.name == 'posix': - fname = '/etc/hosts' - elif os.name == 'nt': # pragma: no cover - fname = os.path.expandvars( - r'%SystemRoot%\system32\drivers\etc\hosts') - self.fname = fname - assert self.fname - self._last_load = 0 - - - def _readlines(self): - # Read the contents of the hosts file. - # - # Return list of lines, comment lines and empty lines are - # excluded. Note that this performs disk I/O so can be - # blocking. - with open(self.fname, 'rb') as fp: - fdata = fp.read() - - - # XXX: Using default decoding. Is that correct? - udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata - - return self.LINES_RE.findall(udata) - def load(self): # pylint:disable=too-many-locals - # Load hosts file - - # This will (re)load the data from the hosts - # file if it has changed. - - try: - load_time = os.stat(self.fname).st_mtime - needs_load = load_time > self._last_load - except (IOError, OSError): - from gevent import get_hub - get_hub().handle_error(self, *sys.exc_info()) - needs_load = False - - if not needs_load: - return - - v4 = {} - v6 = {} - aliases = {} - reverse = {} - - for line in self._readlines(): - parts = line.split() - if len(parts) < 2: - continue - ip = parts.pop(0) - if _is_ipv4_addr(ip): - ipmap = v4 - elif _is_ipv6_addr(ip): - if ip.startswith('fe80'): - # Do not use link-local addresses, OSX stores these here - continue - ipmap = v6 - else: - continue - cname = parts.pop(0).lower() - ipmap[cname] = ip - for alias in parts: - alias = alias.lower() - ipmap[alias] = ip - aliases[alias] = cname - - # XXX: This is wrong for ipv6 - if ipmap is v4: - ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa' - else: - ptr = ip + '.ip6.arpa.' - if ptr not in reverse: - reverse[ptr] = cname - - self._last_load = load_time - self.v4 = v4 - self.v6 = v6 - self.aliases = aliases - self.reverse = reverse - - def iter_all_host_addr_pairs(self): - self.load() - for name, addr in iteritems(self.v4): - yield name, addr - for name, addr in iteritems(self.v6): - yield name, addr class _HostsAnswer(dns.resolver.Answer): # Answer class for HostsResolver object @@ -536,7 +407,7 @@ def _getaliases(self, hostname, family): def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): if ((host in (u'localhost', b'localhost') - or (_is_ipv6_addr(host) and host.startswith('fe80'))) + or (is_ipv6_addr(host) and host.startswith('fe80'))) or not isinstance(host, str) or (flags & AI_NUMERICHOST)): # this handles cases which do not require network access # 1) host is None diff --git a/src/gevent/testing/modules.py b/src/gevent/testing/modules.py index 9e29111d6..3a357e896 100644 --- a/src/gevent/testing/modules.py +++ b/src/gevent/testing/modules.py @@ -29,14 +29,19 @@ from . import util -OPTIONAL_MODULES = [ +OPTIONAL_MODULES = frozenset({ + ## Resolvers. + # ares might not be built 'gevent.resolver_ares', 'gevent.resolver.ares', + # dnspython might not be installed + 'gevent.resolver.dnspython', + ## Backends 'gevent.libev', 'gevent.libev.watcher', 'gevent.libuv.loop', 'gevent.libuv.watcher', -] +}) def walk_modules( @@ -45,6 +50,7 @@ def walk_modules( include_so=False, recursive=False, check_optional=True, + optional_modules=OPTIONAL_MODULES, ): """ Find gevent modules, yielding tuples of ``(path, importable_module_name)``. @@ -53,7 +59,7 @@ def walk_modules( module that is known to be optional on this system (such as a backend), we will attempt to import it; if the import fails, it will not be returned. If false, then we will not make such an attempt, the caller will need to be prepared - for an `ImportError`; the caller can examine *OPTIONAL_MODULES* against + for an `ImportError`; the caller can examine *optional_modules* against the yielded *importable_module_name*. """ # pylint:disable=too-many-branches @@ -78,7 +84,8 @@ def walk_modules( if os.path.exists(pkg_init): yield pkg_init, modpath + fn for p, m in walk_modules(path, modpath + fn + ".", - check_optional=check_optional): + check_optional=check_optional, + optional_modules=optional_modules): yield p, m continue @@ -90,7 +97,7 @@ def walk_modules( 'corecffi', '_corecffi', '_corecffi_build']: continue modname = modpath + x - if check_optional and modname in OPTIONAL_MODULES: + if check_optional and modname in optional_modules: try: with warnings.catch_warnings(): warnings.simplefilter('ignore', DeprecationWarning) diff --git a/src/gevent/tests/test___config.py b/src/gevent/tests/test___config.py index 316e03a8c..382ffd4a0 100644 --- a/src/gevent/tests/test___config.py +++ b/src/gevent/tests/test___config.py @@ -42,9 +42,16 @@ def test_env(self): self.assertEqual(conf.get(), Resolver) # A new object reflects it - conf = _config.Resolver() - from gevent.resolver.dnspython import Resolver as DResolver - self.assertEqual(conf.get(), DResolver) + try: + from gevent.resolver.dnspython import Resolver as DResolver + except ImportError: # pragma: no cover + # dnspython is optional; skip it. + import warnings + warnings.warn('dnspython not installed') + else: + conf = _config.Resolver() + + self.assertEqual(conf.get(), DResolver) def test_set_str_long(self): from gevent.resolver.blocking import Resolver diff --git a/src/gevent/tests/test__all__.py b/src/gevent/tests/test__all__.py index d2744cb12..794557721 100644 --- a/src/gevent/tests/test__all__.py +++ b/src/gevent/tests/test__all__.py @@ -40,15 +40,17 @@ def __contains__(self, item): # helpers NO_ALL = { 'gevent.threading', - 'gevent._util', 'gevent._compat', - 'gevent._socketcommon', + 'gevent._corecffi', + 'gevent._ffi', 'gevent._fileobjectcommon', 'gevent._fileobjectposix', - 'gevent._tblib', - 'gevent._corecffi', 'gevent._patcher', - 'gevent._ffi', + 'gevent._socketcommon', + 'gevent._tblib', + 'gevent._util', + 'gevent.resolver._addresses', + 'gevent.resolver._hostsfile', } ALLOW_IMPLEMENTS = [ @@ -229,7 +231,9 @@ def _test(self, modname): self.module = importlib.import_module(modname) except ImportError: if modname in modules.OPTIONAL_MODULES: - raise unittest.SkipTest("Unable to import %s" % modname) + msg = "Unable to import %s" % modname + warnings.warn(msg) # make the testrunner print it + raise unittest.SkipTest(msg) raise self.check_all() diff --git a/src/gevent/tests/test__socket_dns.py b/src/gevent/tests/test__socket_dns.py index 65c240526..dfc74227f 100644 --- a/src/gevent/tests/test__socket_dns.py +++ b/src/gevent/tests/test__socket_dns.py @@ -467,7 +467,7 @@ def test__broadcast__gethostbyaddr(self): add(TestBroadcast, '') -from gevent.resolver.dnspython import HostsFile # XXX: This will move. +from gevent.resolver._hostsfile import HostsFile class SanitizedHostsFile(HostsFile): def iter_all_host_addr_pairs(self): for name, addr in super(SanitizedHostsFile, self).iter_all_host_addr_pairs():