Merge pull request #1418 from gevent/issue1410

Make dnspython optional for testing.

src/gevent/resolver/_addresses.py

@@ -0,0 +1,163 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019  gevent contributors. See LICENSE for details.
+#
+# Portions of this code taken from dnspython
+#   https://github.com/rthalley/dnspython
+#
+# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license
+
+# Copyright (C) 2003-2017 Nominum, Inc.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose with or without fee is hereby granted,
+# provided that the above copyright notice and this permission notice
+# appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+"""
+Private support for parsing textual addresses.
+
+"""
+from __future__ import absolute_import, division, print_function
+
+import binascii
+import re
+
+from gevent.resolver import hostname_types
+
+
+class AddressSyntaxError(ValueError):
+    pass
+
+
+def _ipv4_inet_aton(text):
+    """
+    Convert an IPv4 address in text form to binary struct.
+
+    *text*, a ``text``, the IPv4 address in textual form.
+
+    Returns a ``binary``.
+    """
+
+    if not isinstance(text, bytes):
+        text = text.encode()
+    parts = text.split(b'.')
+    if len(parts) != 4:
+        raise AddressSyntaxError(text)
+    for part in parts:
+        if not part.isdigit():
+            raise AddressSyntaxError
+        if len(part) > 1 and part[0] == '0':
+            # No leading zeros
+            raise AddressSyntaxError(text)
+    try:
+        ints = [int(part) for part in parts]
+        return struct.pack('BBBB', *ints)
+    except:
+        raise AddressSyntaxError(text)
+
+
+def _ipv6_inet_aton(text,
+                    _v4_ending=re.compile(br'(.*):(\d+\.\d+\.\d+\.\d+)$'),
+                    _colon_colon_start=re.compile(br'::.*'),
+                    _colon_colon_end=re.compile(br'.*::$')):
+    """
+    Convert an IPv6 address in text form to binary form.
+
+    *text*, a ``text``, the IPv6 address in textual form.
+
+    Returns a ``binary``.
+    """
+    # pylint:disable=too-many-branches
+
+    #
+    # Our aim here is not something fast; we just want something that works.
+    #
+    if not isinstance(text, bytes):
+        text = text.encode()
+
+    if text == b'::':
+        text = b'0::'
+    #
+    # Get rid of the icky dot-quad syntax if we have it.
+    #
+    m = _v4_ending.match(text)
+    if not m is None:
+        b = bytearray(_ipv4_inet_aton(m.group(2)))
+        text = (u"{}:{:02x}{:02x}:{:02x}{:02x}".format(m.group(1).decode(),
+                                                       b[0], b[1], b[2],
+                                                       b[3])).encode()
+    #
+    # Try to turn '::<whatever>' into ':<whatever>'; if no match try to
+    # turn '<whatever>::' into '<whatever>:'
+    #
+    m = _colon_colon_start.match(text)
+    if not m is None:
+        text = text[1:]
+    else:
+        m = _colon_colon_end.match(text)
+        if not m is None:
+            text = text[:-1]
+    #
+    # Now canonicalize into 8 chunks of 4 hex digits each
+    #
+    chunks = text.split(b':')
+    l = len(chunks)
+    if l > 8:
+        raise SyntaxError
+    seen_empty = False
+    canonical = []
+    for c in chunks:
+        if c == b'':
+            if seen_empty:
+                raise AddressSyntaxError(text)
+            seen_empty = True
+            for _ in range(0, 8 - l + 1):
+                canonical.append(b'0000')
+        else:
+            lc = len(c)
+            if lc > 4:
+                raise AddressSyntaxError(text)
+            if lc != 4:
+                c = (b'0' * (4 - lc)) + c
+            canonical.append(c)
+    if l < 8 and not seen_empty:
+        raise AddressSyntaxError(text)
+    text = b''.join(canonical)
+
+    #
+    # Finally we can go to binary.
+    #
+    try:
+        return binascii.unhexlify(text)
+    except (binascii.Error, TypeError):
+        raise AddressSyntaxError(text)
+
+
+def _is_addr(host, parse=_ipv4_inet_aton):
+    if not host:
+        return False
+    assert isinstance(host, hostname_types), repr(host)
+    try:
+        parse(host)
+    except AddressSyntaxError:
+        return False
+    else:
+        return True
+
+# Return True if host is a valid IPv4 address
+is_ipv4_addr = _is_addr
+
+
+def is_ipv6_addr(host):
+    # Return True if host is a valid IPv6 address
+    if host:
+        s = '%' if isinstance(host, str) else b'%'
+        host = host.split(s, 1)[0]
+    return _is_addr(host, _ipv6_inet_aton)

src/gevent/resolver/_hostsfile.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019  gevent contributors. See LICENSE for details.
+#
+# Portions of this code taken from dnspython
+#   https://github.com/rthalley/dnspython
+#
+# Copyright (C) Dnspython Contributors, see LICENSE for text of ISC license
+
+# Copyright (C) 2003-2017 Nominum, Inc.
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose with or without fee is hereby granted,
+# provided that the above copyright notice and this permission notice
+# appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND NOMINUM DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL NOMINUM BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
+# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+"""
+Private support for parsing /etc/hosts.
+
+"""
+from __future__ import absolute_import, division, print_function
+
+import sys
+import os
+import re
+
+from gevent.resolver._addresses import is_ipv4_addr
+from gevent.resolver._addresses import is_ipv6_addr
+
+from gevent._compat import iteritems
+
+
+class HostsFile(object):
+    """
+    A class to read the contents of a hosts file (/etc/hosts).
+    """
+
+    LINES_RE = re.compile(r"""
+        \s*  # Leading space
+        ([^\r\n#]+?)  # The actual match, non-greedy so as not to include trailing space
+        \s*  # Trailing space
+        (?:[#][^\r\n]+)?  # Comments
+        (?:$|[\r\n]+)  # EOF or newline
+    """, re.VERBOSE)
+
+    def __init__(self, fname=None):
+        self.v4 = {} # name -> ipv4
+        self.v6 = {} # name -> ipv6
+        self.aliases = {} # name -> canonical_name
+        self.reverse = {} # ip addr -> some name
+        if fname is None:
+            if os.name == 'posix':
+                fname = '/etc/hosts'
+            elif os.name == 'nt': # pragma: no cover
+                fname = os.path.expandvars(
+                    r'%SystemRoot%\system32\drivers\etc\hosts')
+        self.fname = fname
+        assert self.fname
+        self._last_load = 0
+
+
+    def _readlines(self):
+        # Read the contents of the hosts file.
+        #
+        # Return list of lines, comment lines and empty lines are
+        # excluded. Note that this performs disk I/O so can be
+        # blocking.
+        with open(self.fname, 'rb') as fp:
+            fdata = fp.read()
+
+
+        # XXX: Using default decoding. Is that correct?
+        udata = fdata.decode(errors='ignore') if not isinstance(fdata, str) else fdata
+
+        return self.LINES_RE.findall(udata)
+
+    def load(self): # pylint:disable=too-many-locals
+        # Load hosts file
+
+        # This will (re)load the data from the hosts
+        # file if it has changed.
+
+        try:
+            load_time = os.stat(self.fname).st_mtime
+            needs_load = load_time > self._last_load
+        except (IOError, OSError):
+            from gevent import get_hub
+            get_hub().handle_error(self, *sys.exc_info())
+            needs_load = False
+
+        if not needs_load:
+            return
+
+        v4 = {}
+        v6 = {}
+        aliases = {}
+        reverse = {}
+
+        for line in self._readlines():
+            parts = line.split()
+            if len(parts) < 2:
+                continue
+            ip = parts.pop(0)
+            if is_ipv4_addr(ip):
+                ipmap = v4
+            elif is_ipv6_addr(ip):
+                if ip.startswith('fe80'):
+                    # Do not use link-local addresses, OSX stores these here
+                    continue
+                ipmap = v6
+            else:
+                continue
+            cname = parts.pop(0).lower()
+            ipmap[cname] = ip
+            for alias in parts:
+                alias = alias.lower()
+                ipmap[alias] = ip
+                aliases[alias] = cname
+
+            # XXX: This is wrong for ipv6
+            if ipmap is v4:
+                ptr = '.'.join(reversed(ip.split('.'))) + '.in-addr.arpa'
+            else:
+                ptr = ip + '.ip6.arpa.'
+            if ptr not in reverse:
+                reverse[ptr] = cname
+
+        self._last_load = load_time
+        self.v4 = v4
+        self.v6 = v6
+        self.aliases = aliases
+        self.reverse = reverse
+
+    def iter_all_host_addr_pairs(self):
+        self.load()
+        for name, addr in iteritems(self.v4):
+            yield name, addr
+        for name, addr in iteritems(self.v6):
+            yield name, addr