From b5503bbfa7ac91f075f228253f89bb3263a572e0 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 22 Sep 2016 03:34:03 -0700 Subject: [PATCH 01/34] interface proposal --- src/twisted/internet/address.py | 13 ++++ src/twisted/internet/interfaces.py | 102 +++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+) diff --git a/src/twisted/internet/address.py b/src/twisted/internet/address.py index 2b6338ee4c9..5c9f4e821cf 100644 --- a/src/twisted/internet/address.py +++ b/src/twisted/internet/address.py @@ -79,8 +79,21 @@ class IPv6Address(_IPAddress): @ivar host: A string containing a colon-separated, hexadecimal formatted IPv6 address; for example, "::1". @type host: C{str} + + @ivar flowInfo: the IPv6 flow label. This can be used by QoS routers to + identify flows of traffic; you may generally safely ignore it. + @type flowInfo: L{int} + + @ivar scopeID: the IPv6 scope identifier - roughly analagous to what + interface traffic destined for this address must be transmitted over. + @type scopeID: L{int} """ + def __init__(self, type, host, port, flowInfo=0, scopeID=0): + super(IPv6Address, self).__init__(type, host, port) + self.flowInfo = flowInfo + self.scopeID = scopeID + @implementer(IAddress) diff --git a/src/twisted/internet/interfaces.py b/src/twisted/internet/interfaces.py index f6fa9589926..f574fe6804a 100644 --- a/src/twisted/internet/interfaces.py +++ b/src/twisted/internet/interfaces.py @@ -84,6 +84,108 @@ def getHostByName(name, timeout = (1, 3, 11, 45)): +class IHostResolution(Interface): + """ + An L{IHostResolution} represents represents an in-progress recursive query + for a DNS name. + + @since: 16.5 + """ + + name = Attribute( + """ + L{unicode}; the name of the host being resolved. + """ + ) + + def cancel(): + """ + Stop the hostname resolution in progress. + """ + + + +class IResolutionReceiver(Interface): + """ + An L{IResolutionReceiver} receives the results of a hostname resolution in + progress, initiated by an L{IHostnameResolver}. + + @since: 16.5 + """ + + def resolutionBegan(resolutionInProgress): + """ + A hostname resolution began. + + @param resolutionInProgress: an L{IHostResolution}. + """ + + + def addressResolved(address): + """ + An internet address. This is called when an address for the given name + is discovered. In the current implementation this practically means + L{IPv4Address} or L{IPv6Address}, but implementations of this interface + should be lenient to other types being passed to this interface as + well, for future-proofing. + + @param address: An address object. + @type address: L{IAddress} + """ + + + def resolutionComplete(): + """ + Resolution has completed; no further addresses will be relayed to + L{IResolutionReceiver.addressResolved}. + """ + + + +class IHostnameResolver(Interface): + """ + An L{IHostnameResolver} can resolve a host name and port number into a + series of L{IAddress} objects. + + @since: 16.5 + """ + + def resolveHostName(resolutionReceiver, hostName, portNumber=0, + addressTypes=None, transportSemantics='TCP'): + """ + Initiate a hostname resolution. + + @param resolutionReceiver: an object that will receive each resolved + address as it arrives. + @type resolutionReceiver: L{IResolutionReceiver} + + @param hostName: The name of the host to resolve. If this contains + non-ASCII code points, they will be converted to IDNA first. + @type hostName: L{unicode} + + @param portNumber: The port number that the returned addresses should + include. + @type portNumber: L{int} greater than or equal to 0 and less than 65536 + + @param addressTypes: An iterable of implementors of L{IAddress} that + are acceptable values for C{resolutionReceiver} to receive to its + L{addressResolved }. In + practice, this means an iterable containing + L{twisted.internet.address.IPv4Address}, + L{twisted.internet.address.IPv6Address}, both, or neither. + @type addressTypes: L{collections.Iterable} of L{type} + + @param transportSemantics: A string describing the semantics of the + transport; either C{'TCP'} for stream-oriented transports or + C{'UDP'} for datagram-oriented; see + L{twisted.internet.address.IPv6Address.type} and + L{twisted.internet.address.IPv4Address.type}. + @type transportSemantics: native L{str} + """ + + + + class IResolver(IResolverSimple): def query(query, timeout=None): """ From f493f553761840433203732ce1765fb112ed2996 Mon Sep 17 00:00:00 2001 From: Glyph Date: Mon, 17 Oct 2016 23:04:24 -0700 Subject: [PATCH 02/34] wtb $NEXTVER --- src/twisted/internet/interfaces.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/twisted/internet/interfaces.py b/src/twisted/internet/interfaces.py index f574fe6804a..12482317b11 100644 --- a/src/twisted/internet/interfaces.py +++ b/src/twisted/internet/interfaces.py @@ -89,7 +89,7 @@ class IHostResolution(Interface): An L{IHostResolution} represents represents an in-progress recursive query for a DNS name. - @since: 16.5 + @since: 16.6 """ name = Attribute( @@ -110,7 +110,7 @@ class IResolutionReceiver(Interface): An L{IResolutionReceiver} receives the results of a hostname resolution in progress, initiated by an L{IHostnameResolver}. - @since: 16.5 + @since: 16.6 """ def resolutionBegan(resolutionInProgress): @@ -147,7 +147,7 @@ class IHostnameResolver(Interface): An L{IHostnameResolver} can resolve a host name and port number into a series of L{IAddress} objects. - @since: 16.5 + @since: 16.6 """ def resolveHostName(resolutionReceiver, hostName, portNumber=0, From 4d4330b1a9bb1ae337b9b0aa6f303d8bf629db63 Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 18 Oct 2016 01:08:27 -0700 Subject: [PATCH 03/34] document return type --- src/twisted/internet/interfaces.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/twisted/internet/interfaces.py b/src/twisted/internet/interfaces.py index 12482317b11..96795548f14 100644 --- a/src/twisted/internet/interfaces.py +++ b/src/twisted/internet/interfaces.py @@ -181,6 +181,9 @@ def resolveHostName(resolutionReceiver, hostName, portNumber=0, L{twisted.internet.address.IPv6Address.type} and L{twisted.internet.address.IPv4Address.type}. @type transportSemantics: native L{str} + + @return: The resolution in progress. + @rtype: L{IResolutionReceiver} """ From a28268e263b8a1f02e2e24fa7873cfac358cb239 Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 18 Oct 2016 01:08:52 -0700 Subject: [PATCH 04/34] enough ceremony to deterministically call a function --- src/twisted/internet/_resolver.py | 78 +++++++++ src/twisted/internet/test/test_resolver.py | 186 +++++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 src/twisted/internet/_resolver.py create mode 100644 src/twisted/internet/test/test_resolver.py diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py new file mode 100644 index 00000000000..d4b10e75be8 --- /dev/null +++ b/src/twisted/internet/_resolver.py @@ -0,0 +1,78 @@ +# -*- test-case-name: twisted.internet.test.test_resolver -*- +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +IPv6-aware hostname resolution. + +@see: L{IHostnameResolver} +""" + +from __future__ import division, absolute_import + +__metaclass__ = type + +from socket import getaddrinfo + +from zope.interface import implementer +from twisted.internet.interfaces import IHostnameResolver, IHostResolution +from twisted.internet.threads import deferToThreadPool +from twisted.internet.address import IPv4Address + + +@implementer(IHostResolution) +class HostResolution(object): + """ + The in-progress resolution of a given hostname. + """ + + def __init__(self, name): + """ + Create a L{HostResolution} with the given name. + """ + self.name = name + + +@implementer(IHostnameResolver) +class GAIResolver(object): + """ + L{IHostnameResolver} implementation that resolves hostnames by calling + L{getaddrinfo} in a thread. + """ + + def __init__(self, reactor, threadpool=None, getaddrinfo=getaddrinfo): + """ + @param reactor: the reactor to schedule result-delivery on + @type reactor: L{IReactorThreads} + + @param threadpool: the thread pool to use for scheduling name + resolutions. If not supplied, the use the given C{reactor}'s + thread pool. + @type threadpool: L{twisted.internet.threads} + + @param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly + parameterized for testing. + @type getaddrinfo: callable with the same signature as L{getaddrinfo} + """ + self._reactor = reactor + self._threadpool = threadpool + self._getaddrinfo = getaddrinfo + + + def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, + addressTypes=None, transportSemantics='TCP'): + """ + @see: L{IHostnameResolver.resolveHostName} + """ + d = deferToThreadPool(self._reactor, self._threadpool, + self._getaddrinfo, hostName, portNumber) + resolution = HostResolution(hostName) + resolutionReceiver.resolutionBegan(resolution) + @d.addCallback + def deliverResults(result): + for family, socktype, proto, cannoname, sockaddr in result: + resolutionReceiver.addressResolved( + IPv4Address('TCP', *sockaddr) + ) + resolutionReceiver.resolutionComplete() + return resolution diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py new file mode 100644 index 00000000000..36e7e68b7c0 --- /dev/null +++ b/src/twisted/internet/test/test_resolver.py @@ -0,0 +1,186 @@ +# Copyright (c) Twisted Matrix Laboratories. +# See LICENSE for details. + +""" +Tests for implementations of L{IHostnameResolver} and their interactions with +reactor implementations. +""" + +from __future__ import division, absolute_import + +__metaclass__ = type + +from collections import defaultdict + +from socket import gaierror, EAI_NONAME, AF_INET, SOCK_STREAM, IPPROTO_TCP +from threading import local, Lock + +from zope.interface import implementer + +from twisted.internet.interfaces import IResolutionReceiver + +from twisted.trial.unittest import ( + SynchronousTestCase as UnitTest +) +from twisted.python.failure import Failure +from twisted.logger import Logger + +from twisted.python.threadpool import ThreadPool +from twisted._threads import createMemoryWorker, Team, LockWorker + +from twisted.internet.address import IPv4Address +from twisted.internet._resolver import GAIResolver + + +class DeterministicThreadPool(ThreadPool): + """ + Create a deterministic L{ThreadPool} object. + """ + def __init__(self, team): + """ + + """ + self.min = 1 + self.max = 1 + self.name = None + self.threads = [] + self._team = team + + +errorLogger = Logger() + +def deterministicPool(): + """ + Create a deterministic threadpool. + + @return: 2-tuple of L{ThreadPool}, 0-argument C{work} callable; when + C{work} is called, do the work. + """ + worker, doer = createMemoryWorker() + def logIt(): + failure = Failure() + errorLogger.failure("thread call failed", failure) + return ( + DeterministicThreadPool(Team(LockWorker(Lock(), local()), + (lambda: worker), logIt)), + doer + ) + + + +def deterministicReactorThreads(): + """ + Create a deterministic L{IReactorThreads} + """ + worker, doer = createMemoryWorker() + class CFT(object): + def callFromThread(self, f, *a, **k): + worker.do(lambda: f(*a, **k)) + return CFT(), doer + + + +class FakeAddrInfoGetter(object): + """ + Test object implementing getaddrinfo. + """ + + def __init__(self): + """ + Create a L{FakeAddrInfoGetter}. + """ + self.calls = [] + self.results = defaultdict(list) + + + def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): + """ + Mock for getaddrinfo. + """ + self.calls.append((host, port, family, socktype, proto, flags)) + results = self.results[host] + if results: + return results + else: + raise gaierror(EAI_NONAME, + 'nodename nor servname provided, or not known') + + + def addResultForHost(self, host, + sockaddr, + family=AF_INET, + socktype=SOCK_STREAM, + proto=IPPROTO_TCP, + canonname=b""): + """ + Add a result for a given hostname. + """ + self.results[host].append( + (family, socktype, proto, canonname, sockaddr) + ) + + +@implementer(IResolutionReceiver) +class ResultHolder(object): + """ + A resolution receiver which holds onto the results it received. + """ + _started = False + _ended = False + + def __init__(self, testCase): + """ + Create a L{ResultHolder} with a L{UnitTest}. + """ + self._testCase = testCase + + + def resolutionBegan(self, hostResolution): + """ + Hostname resolution began. + """ + self._started = True + self._resolution = hostResolution + self._addresses = [] + + + def addressResolved(self, address): + """ + An address was resolved. + """ + self._addresses.append(address) + + + def resolutionComplete(self): + """ + Hostname resolution is complete. + """ + self._ended = True + + + +class HostnameResolutionTest(UnitTest): + """ + Tests for hostname resolution. + """ + + def test_resolveOneHost(self): + """ + Resolve an individual host. + """ + pool, worker = deterministicPool() + reactor, reactwork = deterministicReactorThreads() + getter = FakeAddrInfoGetter() + receiver = ResultHolder(self) + resolver = GAIResolver(reactor, pool, getter.getaddrinfo) + getter.addResultForHost(b"sample.example.com", ("4.3.2.1", 0)) + + resolution = resolver.resolveHostName(receiver, u"sample.example.com") + self.assertIdentical(receiver._resolution, resolution) + self.assertEqual(receiver._started, True) + self.assertEqual(receiver._ended, False) + worker() + reactwork() + self.assertEqual(receiver._ended, True) + self.assertEqual(receiver._addresses, + [IPv4Address('TCP', '4.3.2.1', 0)]) From f8e87ab265b2baf692faace2feda1d98d27c1a61 Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 18 Oct 2016 01:25:28 -0700 Subject: [PATCH 05/34] hrm. should this be a native string? --- src/twisted/internet/test/test_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index 36e7e68b7c0..88492a6ef8e 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -173,7 +173,7 @@ def test_resolveOneHost(self): getter = FakeAddrInfoGetter() receiver = ResultHolder(self) resolver = GAIResolver(reactor, pool, getter.getaddrinfo) - getter.addResultForHost(b"sample.example.com", ("4.3.2.1", 0)) + getter.addResultForHost(u"sample.example.com", ("4.3.2.1", 0)) resolution = resolver.resolveHostName(receiver, u"sample.example.com") self.assertIdentical(receiver._resolution, resolution) From ab7db3c4be0dcc88727f34c43578fa629435a220 Mon Sep 17 00:00:00 2001 From: Glyph Date: Wed, 19 Oct 2016 01:17:13 -0700 Subject: [PATCH 06/34] AF_* sensitivity --- src/twisted/internet/_resolver.py | 9 ++-- src/twisted/internet/address.py | 2 + src/twisted/internet/test/test_resolver.py | 63 +++++++++++++++++----- 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index d4b10e75be8..3a188ce5cec 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -12,12 +12,13 @@ __metaclass__ = type -from socket import getaddrinfo +from socket import getaddrinfo, AF_INET, AF_INET6 from zope.interface import implementer + from twisted.internet.interfaces import IHostnameResolver, IHostResolution from twisted.internet.threads import deferToThreadPool -from twisted.internet.address import IPv4Address +from twisted.internet.address import IPv4Address, IPv6Address @implementer(IHostResolution) @@ -71,8 +72,10 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, @d.addCallback def deliverResults(result): for family, socktype, proto, cannoname, sockaddr in result: + addrType = {AF_INET: IPv4Address, + AF_INET6: IPv6Address}[family] resolutionReceiver.addressResolved( - IPv4Address('TCP', *sockaddr) + addrType('TCP', *sockaddr) ) resolutionReceiver.resolutionComplete() return resolution diff --git a/src/twisted/internet/address.py b/src/twisted/internet/address.py index 5c9f4e821cf..2c50b35af77 100644 --- a/src/twisted/internet/address.py +++ b/src/twisted/internet/address.py @@ -89,6 +89,8 @@ class IPv6Address(_IPAddress): @type scopeID: L{int} """ + compareAttributes = ('type', 'host', 'port', 'flowInfo', 'scopeID') + def __init__(self, type, host, port, flowInfo=0, scopeID=0): super(IPv6Address, self).__init__(type, host, port) self.flowInfo = flowInfo diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index 88492a6ef8e..d96dc914c16 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -12,7 +12,9 @@ from collections import defaultdict -from socket import gaierror, EAI_NONAME, AF_INET, SOCK_STREAM, IPPROTO_TCP +from socket import ( + gaierror, EAI_NONAME, AF_INET, AF_INET6, SOCK_STREAM, IPPROTO_TCP +) from threading import local, Lock from zope.interface import implementer @@ -28,7 +30,7 @@ from twisted.python.threadpool import ThreadPool from twisted._threads import createMemoryWorker, Team, LockWorker -from twisted.internet.address import IPv4Address +from twisted.internet.address import IPv4Address, IPv6Address from twisted.internet._resolver import GAIResolver @@ -38,7 +40,7 @@ class DeterministicThreadPool(ThreadPool): """ def __init__(self, team): """ - + Create a L{DeterministicThreadPool} from a L{Team}. """ self.min = 1 self.max = 1 @@ -164,23 +166,58 @@ class HostnameResolutionTest(UnitTest): Tests for hostname resolution. """ + def setUp(self): + """ + Set up a L{GAIResolver}. + """ + self.pool, self.worker = deterministicPool() + self.reactor, self.reactwork = deterministicReactorThreads() + self.getter = FakeAddrInfoGetter() + self.resolver = GAIResolver(self.reactor, self.pool, + self.getter.getaddrinfo) + + def test_resolveOneHost(self): """ - Resolve an individual host. + Resolving an individual hostname that results in one address from + getaddrinfo results in a single call each to C{resolutionBegan}, + C{addressResolved}, and C{resolutionComplete}. """ - pool, worker = deterministicPool() - reactor, reactwork = deterministicReactorThreads() - getter = FakeAddrInfoGetter() receiver = ResultHolder(self) - resolver = GAIResolver(reactor, pool, getter.getaddrinfo) - getter.addResultForHost(u"sample.example.com", ("4.3.2.1", 0)) - - resolution = resolver.resolveHostName(receiver, u"sample.example.com") + self.getter.addResultForHost(u"sample.example.com", ("4.3.2.1", 0)) + resolution = self.resolver.resolveHostName(receiver, + u"sample.example.com") self.assertIdentical(receiver._resolution, resolution) self.assertEqual(receiver._started, True) self.assertEqual(receiver._ended, False) - worker() - reactwork() + self.worker() + self.reactwork() self.assertEqual(receiver._ended, True) self.assertEqual(receiver._addresses, [IPv4Address('TCP', '4.3.2.1', 0)]) + + + def test_resolveOneIPv6Host(self): + """ + Resolving an individual hostname that results in one address from + getaddrinfo results in a single call each to C{resolutionBegan}, + C{addressResolved}, and C{resolutionComplete}; C{addressResolved} will + receive an L{IPv6Address}. + """ + receiver = ResultHolder(self) + flowInfo = 1 + scopeID = 2 + self.getter.addResultForHost(u"sample.example.com", + ("::1", 0, flowInfo, scopeID), + family=AF_INET6) + resolution = self.resolver.resolveHostName(receiver, + u"sample.example.com") + self.assertIdentical(receiver._resolution, resolution) + self.assertEqual(receiver._started, True) + self.assertEqual(receiver._ended, False) + self.worker() + self.reactwork() + self.assertEqual(receiver._ended, True) + self.assertEqual(receiver._addresses, + [IPv6Address('TCP', '::1', 0, flowInfo, scopeID)]) + From 590033949658ef73b632d66b8364a45b23cc465b Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 29 Oct 2016 00:04:17 -0700 Subject: [PATCH 07/34] topfile --- src/twisted/topfiles/4362.feature | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/twisted/topfiles/4362.feature diff --git a/src/twisted/topfiles/4362.feature b/src/twisted/topfiles/4362.feature new file mode 100644 index 00000000000..0baa6da1a6e --- /dev/null +++ b/src/twisted/topfiles/4362.feature @@ -0,0 +1,4 @@ +Added a new interface, twisted.internet.interfaces.IHostnameResolver, which is +an improvement to twisted.internet.interfaces.IResolverSimple that supports +resolving multiple addresses as well as resolving IPv6 addresses. This is a +native, asynchronous, Twisted analogue to getaddrinfo. From 3ef18ce7b600c4bc0e57fc0b37d99458abc93583 Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 29 Oct 2016 00:06:00 -0700 Subject: [PATCH 08/34] new style --- src/twisted/internet/test/test_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index d96dc914c16..f916519799c 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -34,7 +34,7 @@ from twisted.internet._resolver import GAIResolver -class DeterministicThreadPool(ThreadPool): +class DeterministicThreadPool(ThreadPool, object): """ Create a deterministic L{ThreadPool} object. """ From 0b7919ac13907d0a027139cddafc07a3b6b6803e Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 29 Oct 2016 00:23:27 -0700 Subject: [PATCH 09/34] test for error case --- src/twisted/internet/test/test_resolver.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index f916519799c..2b5cda36e19 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -221,3 +221,20 @@ def test_resolveOneIPv6Host(self): self.assertEqual(receiver._addresses, [IPv6Address('TCP', '::1', 0, flowInfo, scopeID)]) + + def test_gaierror(self): + """ + Resolving a hostname that results in C{getaddrinfo} raising a + L{gaierror} will result in the L{IResolutionReceiver} receiving a call + to C{resolutionComplete} with no C{addressResolved} calls in between; + no failure is logged. + """ + receiver = ResultHolder(self) + resolution = self.resolver.resolveHostName(receiver, + u"sample.example.com") + self.assertIs(receiver._resolution, resolution) + self.worker() + self.reactwork() + self.assertEqual(receiver._started, True) + self.assertEqual(receiver._ended, True) + self.assertEqual(receiver._addresses, []) From 1a02f09c01848f7a62d5ad3954502ce7c974d514 Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 29 Oct 2016 00:23:54 -0700 Subject: [PATCH 10/34] assertIs --- src/twisted/internet/test/test_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index 2b5cda36e19..00267d3dd85 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -187,7 +187,7 @@ def test_resolveOneHost(self): self.getter.addResultForHost(u"sample.example.com", ("4.3.2.1", 0)) resolution = self.resolver.resolveHostName(receiver, u"sample.example.com") - self.assertIdentical(receiver._resolution, resolution) + self.assertIs(receiver._resolution, resolution) self.assertEqual(receiver._started, True) self.assertEqual(receiver._ended, False) self.worker() @@ -212,7 +212,7 @@ def test_resolveOneIPv6Host(self): family=AF_INET6) resolution = self.resolver.resolveHostName(receiver, u"sample.example.com") - self.assertIdentical(receiver._resolution, resolution) + self.assertIs(receiver._resolution, resolution) self.assertEqual(receiver._started, True) self.assertEqual(receiver._ended, False) self.worker() From 6f6525279330a4038561b111bd56afc361e6ae4a Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 29 Oct 2016 00:26:40 -0700 Subject: [PATCH 11/34] implement the error case --- src/twisted/internet/_resolver.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 3a188ce5cec..b8fed46156b 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -12,7 +12,7 @@ __metaclass__ = type -from socket import getaddrinfo, AF_INET, AF_INET6 +from socket import getaddrinfo, AF_INET, AF_INET6, gaierror from zope.interface import implementer @@ -65,8 +65,12 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, """ @see: L{IHostnameResolver.resolveHostName} """ - d = deferToThreadPool(self._reactor, self._threadpool, - self._getaddrinfo, hostName, portNumber) + def get(): + try: + return self._getaddrinfo(hostName, portNumber) + except gaierror: + return [] + d = deferToThreadPool(self._reactor, self._threadpool, get) resolution = HostResolution(hostName) resolutionReceiver.resolutionBegan(resolution) @d.addCallback From 3c1a225fa9baa79c4988370f466b90195b8a3273 Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 29 Oct 2016 16:13:24 -0700 Subject: [PATCH 12/34] write a test for code that codecov's complaining about and hey, what do you know, it's dead code that's unnecessary. --- src/twisted/internet/test/test_resolver.py | 27 +++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index 00267d3dd85..a65fd27eb81 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -49,8 +49,6 @@ def __init__(self, team): self._team = team -errorLogger = Logger() - def deterministicPool(): """ Create a deterministic threadpool. @@ -59,12 +57,9 @@ def deterministicPool(): C{work} is called, do the work. """ worker, doer = createMemoryWorker() - def logIt(): - failure = Failure() - errorLogger.failure("thread call failed", failure) return ( DeterministicThreadPool(Team(LockWorker(Lock(), local()), - (lambda: worker), logIt)), + (lambda: worker), lambda: None)), doer ) @@ -161,6 +156,26 @@ def resolutionComplete(self): +class HelperTests(UnitTest): + """ + Tests for error cases of helpers used in this module. + """ + + def test_logErrorsInThreads(self): + """ + L{DeterministicThreadPool} will log any exceptions that its "thread" + workers encounter. + """ + self.pool, self.worker = deterministicPool() + def divideByZero(): + return 1 / 0 + self.pool.callInThread(divideByZero) + self.worker() + self.assertEqual(len(self.flushLoggedErrors(ZeroDivisionError)), 1) + + + + class HostnameResolutionTest(UnitTest): """ Tests for hostname resolution. From 53c10abb8f3d44befd9342baf5326f1d7e2e8d18 Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 29 Oct 2016 16:17:45 -0700 Subject: [PATCH 13/34] oops --- src/twisted/internet/test/test_resolver.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index a65fd27eb81..2eab909e295 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -24,8 +24,6 @@ from twisted.trial.unittest import ( SynchronousTestCase as UnitTest ) -from twisted.python.failure import Failure -from twisted.logger import Logger from twisted.python.threadpool import ThreadPool from twisted._threads import createMemoryWorker, Team, LockWorker From 8da4f9cde4fa837760f737b1c0312100b324a613 Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 12 Nov 2016 21:56:25 -0800 Subject: [PATCH 14/34] interface doc updates --- src/twisted/internet/interfaces.py | 40 +++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/twisted/internet/interfaces.py b/src/twisted/internet/interfaces.py index 96795548f14..705ece2422f 100644 --- a/src/twisted/internet/interfaces.py +++ b/src/twisted/internet/interfaces.py @@ -89,7 +89,7 @@ class IHostResolution(Interface): An L{IHostResolution} represents represents an in-progress recursive query for a DNS name. - @since: 16.6 + @since: 16.7 """ name = Attribute( @@ -110,7 +110,7 @@ class IResolutionReceiver(Interface): An L{IResolutionReceiver} receives the results of a hostname resolution in progress, initiated by an L{IHostnameResolver}. - @since: 16.6 + @since: 16.7 """ def resolutionBegan(resolutionInProgress): @@ -147,7 +147,7 @@ class IHostnameResolver(Interface): An L{IHostnameResolver} can resolve a host name and port number into a series of L{IAddress} objects. - @since: 16.6 + @since: 16.7 """ def resolveHostName(resolutionReceiver, hostName, portNumber=0, @@ -1518,9 +1518,14 @@ def callWhenRunning(callable, *args, **kw): """ + class IReactorPluggableResolver(Interface): """ - A reactor with a pluggable name resolver interface. + An L{IReactorPluggableResolver} is a reactor which can be customized with + an L{IResolverSimple}. This is a fairly limited interface, that supports + only IPv4; you should use L{IReactorPluggableNameResolver} instead. + + @see: L{IReactorPluggableNameResolver} """ def installResolver(resolver): @@ -1531,9 +1536,36 @@ def installResolver(resolver): @param resolver: The new resolver to use. @return: The previously installed resolver. + @rtype: L{IResolverSimple} """ + +class IReactorPluggableNameResolver(Interface): + """ + An L{IReactorPluggableNameResolver} is a reactor whose name resolver can be + set to a user-supplied object. + """ + + resolver = Attribute( + """ + Read-only attribute; the resolver installed with L{installResolver}. + """ + ) + + def installResolver(resolver): + """ + Set the internal resolver to use for name lookups. + + @type resolver: An object implementing the L{IResolverSimple} interface + @param resolver: The new resolver to use. + + @return: The previously installed resolver. + @rtype: L{IHostnameResolver} + """ + + + class IReactorDaemonize(Interface): """ A reactor which provides hooks that need to be called before and after From ccf377ff5eeb46c8b3f324a030a5aa8c1f8c8a8d Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 18:22:18 -0800 Subject: [PATCH 15/34] address family selection --- src/twisted/internet/test/test_resolver.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index 2eab909e295..c66450baca5 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -251,3 +251,19 @@ def test_gaierror(self): self.assertEqual(receiver._started, True) self.assertEqual(receiver._ended, True) self.assertEqual(receiver._addresses, []) + + + def test_resolveOnlyIPv4(self): + """ + When passed an C{addressTypes} parameter containing only + L{IPv4Address}, L{GAIResolver} will pass C{AF_INET} to C{getaddrinfo}. + """ + receiver = ResultHolder(self) + resolution = self.resolver.resolveHostName( + receiver, u"sample.example.com", addressTypes=[IPv4Address] + ) + self.assertIs(receiver._resolution, resolution) + self.worker() + self.reactwork() + host, port, family, socktype, proto, flags = self.getter.calls[0] + self.assertEqual(socktype, AF_INET) From d9d48ec7d5b3dc768093f522d692a37e60f894a0 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 20:27:28 -0800 Subject: [PATCH 16/34] docstring adjustment --- src/twisted/internet/_resolver.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index b8fed46156b..581225768f5 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -43,6 +43,8 @@ class GAIResolver(object): def __init__(self, reactor, threadpool=None, getaddrinfo=getaddrinfo): """ + Create a L{GAIResolver}. + @param reactor: the reactor to schedule result-delivery on @type reactor: L{IReactorThreads} @@ -62,9 +64,6 @@ def __init__(self, reactor, threadpool=None, getaddrinfo=getaddrinfo): def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics='TCP'): - """ - @see: L{IHostnameResolver.resolveHostName} - """ def get(): try: return self._getaddrinfo(hostName, portNumber) From 6795570e43553feffb10b34a950b98fa20d601c2 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 20:38:47 -0800 Subject: [PATCH 17/34] pacify twistedchecker --- src/twisted/internet/_resolver.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 581225768f5..139d055240b 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -34,6 +34,7 @@ def __init__(self, name): self.name = name + @implementer(IHostnameResolver) class GAIResolver(object): """ @@ -64,6 +65,21 @@ def __init__(self, reactor, threadpool=None, getaddrinfo=getaddrinfo): def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, addressTypes=None, transportSemantics='TCP'): + """ + See L{IHostnameResolver.resolveHostName} + + @param resolutionReceiver: see interface + + @param hostName: see interface + + @param portNumber: see interface + + @param addressTypes: see interface + + @param transportSemantics: see interface + + @return: see interface + """ def get(): try: return self._getaddrinfo(hostName, portNumber) From bcdab97fa9925277ca1b5aa867edf91f6e34722d Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 21:52:30 -0800 Subject: [PATCH 18/34] vws --- src/twisted/internet/test/test_resolver.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index c66450baca5..d87918ba884 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -47,6 +47,7 @@ def __init__(self, team): self._team = team + def deterministicPool(): """ Create a deterministic threadpool. @@ -115,6 +116,7 @@ def addResultForHost(self, host, ) + @implementer(IResolutionReceiver) class ResultHolder(object): """ @@ -173,7 +175,6 @@ def divideByZero(): - class HostnameResolutionTest(UnitTest): """ Tests for hostname resolution. From b59681e6234b705490a1bd52bd141f9d8478cb8f Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:14:38 -0800 Subject: [PATCH 19/34] twistedchecker clean --- src/twisted/internet/test/test_resolver.py | 54 +++++++++++++++++++--- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index d87918ba884..d66ab5041c1 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -67,6 +67,10 @@ def deterministicPool(): def deterministicReactorThreads(): """ Create a deterministic L{IReactorThreads} + + @return: a 2-tuple consisting of an L{IReactorThreads}-like object and a + 0-argument callable that will perform one unit of work invoked via that + object's C{callFromThread} method. """ worker, doer = createMemoryWorker() class CFT(object): @@ -91,7 +95,21 @@ def __init__(self): def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): """ - Mock for getaddrinfo. + Mock for L{socket.getaddrinfo}. + + @param host: see L{socket.getaddrinfo} + + @param port: see L{socket.getaddrinfo} + + @param family: see L{socket.getaddrinfo} + + @param socktype: see L{socket.getaddrinfo} + + @param proto: see L{socket.getaddrinfo} + + @param flags: see L{socket.getaddrinfo} + + @return: L{socket.getaddrinfo} """ self.calls.append((host, port, family, socktype, proto, flags)) results = self.results[host] @@ -102,14 +120,32 @@ def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): 'nodename nor servname provided, or not known') - def addResultForHost(self, host, - sockaddr, - family=AF_INET, - socktype=SOCK_STREAM, - proto=IPPROTO_TCP, + def addResultForHost(self, host, sockaddr, family=AF_INET, + socktype=SOCK_STREAM, proto=IPPROTO_TCP, canonname=b""): """ Add a result for a given hostname. + + @param host: The hostname to give this result for. This will be the + next result from L{FakeAddrInfoGetter.getaddrinfo} when passed this + host. + @type canonname: native L{str} + + @param sockaddr: The resulting socket address; should be a 2-tuple for + IPv4 or a 4-tuple for IPv6. + + @param family: An C{AF_*} constant that will be returned from + C{getaddrinfo}. + + @param socktype: A C{SOCK_*} constant that will be returned from + C{getaddrinfo}. + + @param proto: An C{IPPROTO_*} constant that will be returned from + C{getaddrinfo}. + + @param canonname: A canonical name that will be returned from + C{getaddrinfo}. + @type canonname: native L{str} """ self.results[host].append( (family, socktype, proto, canonname, sockaddr) @@ -135,6 +171,8 @@ def __init__(self, testCase): def resolutionBegan(self, hostResolution): """ Hostname resolution began. + + @param hostResolution: see L{IResolutionReceiver} """ self._started = True self._resolution = hostResolution @@ -144,6 +182,8 @@ def resolutionBegan(self, hostResolution): def addressResolved(self, address): """ An address was resolved. + + @param address: see L{IResolutionReceiver} """ self._addresses.append(address) @@ -175,7 +215,7 @@ def divideByZero(): -class HostnameResolutionTest(UnitTest): +class HostnameResolutionTests(UnitTest): """ Tests for hostname resolution. """ From 4bf56011dcb23e85e963e0360fff9bb8f414b076 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:33:14 -0800 Subject: [PATCH 20/34] tests & fix for various AF_ types --- src/twisted/internet/_resolver.py | 16 +++++++- src/twisted/internet/test/test_resolver.py | 43 +++++++++++++++++++--- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 139d055240b..63ba035cc9e 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -12,7 +12,7 @@ __metaclass__ = type -from socket import getaddrinfo, AF_INET, AF_INET6, gaierror +from socket import getaddrinfo, AF_INET, AF_INET6, AF_UNSPEC, gaierror from zope.interface import implementer @@ -35,6 +35,14 @@ def __init__(self, name): +_typesToAF = { + frozenset([IPv4Address]): AF_INET, + frozenset([IPv6Address]): AF_INET6, + frozenset([IPv4Address, IPv6Address]): AF_UNSPEC, +} + + + @implementer(IHostnameResolver) class GAIResolver(object): """ @@ -80,9 +88,13 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, @return: see interface """ + if addressTypes is None: + addressTypes = [IPv4Address, IPv6Address] + addressTypes = frozenset(addressTypes) + addressFamily = _typesToAF[addressTypes] def get(): try: - return self._getaddrinfo(hostName, portNumber) + return self._getaddrinfo(hostName, portNumber, addressFamily) except gaierror: return [] d = deferToThreadPool(self._reactor, self._threadpool, get) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index d66ab5041c1..a209307320b 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -13,7 +13,8 @@ from collections import defaultdict from socket import ( - gaierror, EAI_NONAME, AF_INET, AF_INET6, SOCK_STREAM, IPPROTO_TCP + gaierror, EAI_NONAME, AF_INET, AF_INET6, AF_UNSPEC, SOCK_STREAM, + IPPROTO_TCP ) from threading import local, Lock @@ -294,17 +295,47 @@ def test_gaierror(self): self.assertEqual(receiver._addresses, []) - def test_resolveOnlyIPv4(self): + def _resolveOnlyTest(self, addrTypes, expectedAF): """ - When passed an C{addressTypes} parameter containing only - L{IPv4Address}, L{GAIResolver} will pass C{AF_INET} to C{getaddrinfo}. + Verify that the given set of address types results in the given C{AF_} + constant being passed to C{getaddrinfo}. + + @param addrTypes: iterable of L{IAddress} implementers + + @param expectedAF: an C{AF_*} constant """ receiver = ResultHolder(self) resolution = self.resolver.resolveHostName( - receiver, u"sample.example.com", addressTypes=[IPv4Address] + receiver, u"sample.example.com", addressTypes=addrTypes ) self.assertIs(receiver._resolution, resolution) self.worker() self.reactwork() host, port, family, socktype, proto, flags = self.getter.calls[0] - self.assertEqual(socktype, AF_INET) + self.assertEqual(family, expectedAF) + + + def test_resolveOnlyIPv4(self): + """ + When passed an C{addressTypes} parameter containing only + L{IPv4Address}, L{GAIResolver} will pass C{AF_INET} to C{getaddrinfo}. + """ + self._resolveOnlyTest([IPv4Address], AF_INET) + + + def test_resolveOnlyIPv6(self): + """ + When passed an C{addressTypes} parameter containing only + L{IPv6Address}, L{GAIResolver} will pass C{AF_INET6} to C{getaddrinfo}. + """ + self._resolveOnlyTest([IPv6Address], AF_INET6) + + + def test_resolveBoth(self): + """ + When passed an C{addressTypes} parameter containing both L{IPv4Address} + and L{IPv6Address} (or the default of C{None}, which carries the same + meaning), L{GAIResolver} will pass C{AF_UNSPEC} to C{getaddrinfo}. + """ + self._resolveOnlyTest([IPv4Address, IPv6Address], AF_UNSPEC) + self._resolveOnlyTest(None, AF_UNSPEC) From 819eed65a6206d4264c09885152aa6752c24d842 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:37:42 -0800 Subject: [PATCH 21/34] hoist constant --- src/twisted/internet/_resolver.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 63ba035cc9e..b1aa3d1e890 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -41,6 +41,11 @@ def __init__(self, name): frozenset([IPv4Address, IPv6Address]): AF_UNSPEC, } +_afToType = { + AF_INET: IPv4Address, + AF_INET6: IPv6Address, +} + @implementer(IHostnameResolver) @@ -103,8 +108,7 @@ def get(): @d.addCallback def deliverResults(result): for family, socktype, proto, cannoname, sockaddr in result: - addrType = {AF_INET: IPv4Address, - AF_INET6: IPv6Address}[family] + addrType = _afToType[family] resolutionReceiver.addressResolved( addrType('TCP', *sockaddr) ) From 537fabb44a42138086156fb3abb259559dc828ca Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:38:07 -0800 Subject: [PATCH 22/34] fewer lines --- src/twisted/internet/_resolver.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index b1aa3d1e890..a4a052b517c 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -95,8 +95,7 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, """ if addressTypes is None: addressTypes = [IPv4Address, IPv6Address] - addressTypes = frozenset(addressTypes) - addressFamily = _typesToAF[addressTypes] + addressFamily = _typesToAF[frozenset(addressTypes)] def get(): try: return self._getaddrinfo(hostName, portNumber, addressFamily) From 86949be3ef8847b39b2dea014ff334b56ef45033 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:42:11 -0800 Subject: [PATCH 23/34] less code more data --- src/twisted/internet/_resolver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index a4a052b517c..c728f118c87 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -39,6 +39,7 @@ def __init__(self, name): frozenset([IPv4Address]): AF_INET, frozenset([IPv6Address]): AF_INET6, frozenset([IPv4Address, IPv6Address]): AF_UNSPEC, + None: AF_UNSPEC, } _afToType = { @@ -93,9 +94,8 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, @return: see interface """ - if addressTypes is None: - addressTypes = [IPv4Address, IPv6Address] - addressFamily = _typesToAF[frozenset(addressTypes)] + addressFamily = _typesToAF[frozenset(addressTypes) + if addressTypes is not None else None] def get(): try: return self._getaddrinfo(hostName, portNumber, addressFamily) From 402778d9aae306ba24c84721ff9403f49124a150 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:42:58 -0800 Subject: [PATCH 24/34] None first --- src/twisted/internet/_resolver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index c728f118c87..9d38d76d628 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -94,8 +94,8 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, @return: see interface """ - addressFamily = _typesToAF[frozenset(addressTypes) - if addressTypes is not None else None] + addressFamily = _typesToAF[None if addressTypes is None + else frozenset(addressTypes)] def get(): try: return self._getaddrinfo(hostName, portNumber, addressFamily) From 8b0718ff1c8eb1e55253a99598a33275badc1e03 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:44:20 -0800 Subject: [PATCH 25/34] no, Nones are bad --- src/twisted/internet/_resolver.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 9d38d76d628..655bbd67605 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -35,11 +35,12 @@ def __init__(self, name): +_any = frozenset([IPv4Address, IPv6Address]) + _typesToAF = { frozenset([IPv4Address]): AF_INET, frozenset([IPv6Address]): AF_INET6, - frozenset([IPv4Address, IPv6Address]): AF_UNSPEC, - None: AF_UNSPEC, + _any: AF_UNSPEC, } _afToType = { @@ -94,7 +95,7 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, @return: see interface """ - addressFamily = _typesToAF[None if addressTypes is None + addressFamily = _typesToAF[_any if addressTypes is None else frozenset(addressTypes)] def get(): try: From efb07fe010aafa735314402150646a90fee19adc Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 22:54:03 -0800 Subject: [PATCH 26/34] socket types --- src/twisted/internet/_resolver.py | 12 +++++- src/twisted/internet/test/test_resolver.py | 50 ++++++++++++++++------ 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 655bbd67605..fab8b05143c 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -12,7 +12,8 @@ __metaclass__ = type -from socket import getaddrinfo, AF_INET, AF_INET6, AF_UNSPEC, gaierror +from socket import (getaddrinfo, AF_INET, AF_INET6, AF_UNSPEC, SOCK_STREAM, + SOCK_DGRAM, gaierror) from zope.interface import implementer @@ -48,6 +49,11 @@ def __init__(self, name): AF_INET6: IPv6Address, } +_transportToSocket = { + 'TCP': SOCK_STREAM, + 'UDP': SOCK_DGRAM, +} + @implementer(IHostnameResolver) @@ -97,9 +103,11 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, """ addressFamily = _typesToAF[_any if addressTypes is None else frozenset(addressTypes)] + socketType = _transportToSocket[transportSemantics] def get(): try: - return self._getaddrinfo(hostName, portNumber, addressFamily) + return self._getaddrinfo(hostName, portNumber, addressFamily, + socketType) except gaierror: return [] d = deferToThreadPool(self._reactor, self._threadpool, get) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index a209307320b..564f11f6ab9 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -14,7 +14,7 @@ from socket import ( gaierror, EAI_NONAME, AF_INET, AF_INET6, AF_UNSPEC, SOCK_STREAM, - IPPROTO_TCP + SOCK_DGRAM, IPPROTO_TCP ) from threading import local, Lock @@ -207,11 +207,11 @@ def test_logErrorsInThreads(self): L{DeterministicThreadPool} will log any exceptions that its "thread" workers encounter. """ - self.pool, self.worker = deterministicPool() + self.pool, self.doThreadWork = deterministicPool() def divideByZero(): return 1 / 0 self.pool.callInThread(divideByZero) - self.worker() + self.doThreadWork() self.assertEqual(len(self.flushLoggedErrors(ZeroDivisionError)), 1) @@ -225,8 +225,8 @@ def setUp(self): """ Set up a L{GAIResolver}. """ - self.pool, self.worker = deterministicPool() - self.reactor, self.reactwork = deterministicReactorThreads() + self.pool, self.doThreadWork = deterministicPool() + self.reactor, self.doReactorWork = deterministicReactorThreads() self.getter = FakeAddrInfoGetter() self.resolver = GAIResolver(self.reactor, self.pool, self.getter.getaddrinfo) @@ -245,8 +245,8 @@ def test_resolveOneHost(self): self.assertIs(receiver._resolution, resolution) self.assertEqual(receiver._started, True) self.assertEqual(receiver._ended, False) - self.worker() - self.reactwork() + self.doThreadWork() + self.doReactorWork() self.assertEqual(receiver._ended, True) self.assertEqual(receiver._addresses, [IPv4Address('TCP', '4.3.2.1', 0)]) @@ -270,8 +270,8 @@ def test_resolveOneIPv6Host(self): self.assertIs(receiver._resolution, resolution) self.assertEqual(receiver._started, True) self.assertEqual(receiver._ended, False) - self.worker() - self.reactwork() + self.doThreadWork() + self.doReactorWork() self.assertEqual(receiver._ended, True) self.assertEqual(receiver._addresses, [IPv6Address('TCP', '::1', 0, flowInfo, scopeID)]) @@ -288,8 +288,8 @@ def test_gaierror(self): resolution = self.resolver.resolveHostName(receiver, u"sample.example.com") self.assertIs(receiver._resolution, resolution) - self.worker() - self.reactwork() + self.doThreadWork() + self.doReactorWork() self.assertEqual(receiver._started, True) self.assertEqual(receiver._ended, True) self.assertEqual(receiver._addresses, []) @@ -309,8 +309,8 @@ def _resolveOnlyTest(self, addrTypes, expectedAF): receiver, u"sample.example.com", addressTypes=addrTypes ) self.assertIs(receiver._resolution, resolution) - self.worker() - self.reactwork() + self.doThreadWork() + self.doReactorWork() host, port, family, socktype, proto, flags = self.getter.calls[0] self.assertEqual(family, expectedAF) @@ -339,3 +339,27 @@ def test_resolveBoth(self): """ self._resolveOnlyTest([IPv4Address, IPv6Address], AF_UNSPEC) self._resolveOnlyTest(None, AF_UNSPEC) + + + def test_transportSemanticsToSocketType(self): + """ + When passed a C{transportSemantics} paramter, C{'TCP'} (the value + present in L{IPv4Address.type} to indicate a stream transport) maps to + C{SOCK_STREAM} and C{'UDP'} maps to C{SOCK_DGRAM}. + """ + receiver = ResultHolder(self) + self.resolver.resolveHostName(receiver, u"example.com", + transportSemantics='TCP') + receiver2 = ResultHolder(self) + self.resolver.resolveHostName(receiver2, u"example.com", + transportSemantics='UDP') + self.doThreadWork() + self.doReactorWork() + self.doThreadWork() + self.doReactorWork() + host, port, family, socktypeT, proto, flags = self.getter.calls[0] + host, port, family, socktypeU, proto, flags = self.getter.calls[1] + self.assertEqual(socktypeT, SOCK_STREAM) + self.assertEqual(socktypeU, SOCK_DGRAM) + + From bd740c70f45b67b9c978d234251ea28e81340d2e Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 23:14:57 -0800 Subject: [PATCH 27/34] socktype --- src/twisted/internet/_resolver.py | 7 ++++- src/twisted/internet/test/test_resolver.py | 31 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index fab8b05143c..4f493f8db23 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -54,6 +54,11 @@ def __init__(self, name): 'UDP': SOCK_DGRAM, } +_socktypeToType = { + SOCK_STREAM: 'TCP', + SOCK_DGRAM: 'UDP', +} + @implementer(IHostnameResolver) @@ -118,7 +123,7 @@ def deliverResults(result): for family, socktype, proto, cannoname, sockaddr in result: addrType = _afToType[family] resolutionReceiver.addressResolved( - addrType('TCP', *sockaddr) + addrType(_socktypeToType.get(socktype, 'TCP'), *sockaddr) ) resolutionReceiver.resolutionComplete() return resolution diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index 564f11f6ab9..b81ed5ec221 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -125,11 +125,14 @@ def addResultForHost(self, host, sockaddr, family=AF_INET, socktype=SOCK_STREAM, proto=IPPROTO_TCP, canonname=b""): """ - Add a result for a given hostname. + Add a result for a given hostname. When this hostname is resolved, the + result will be a L{list} of all results C{addResultForHost} has been + called with using that hostname so far. @param host: The hostname to give this result for. This will be the next result from L{FakeAddrInfoGetter.getaddrinfo} when passed this host. + @type canonname: native L{str} @param sockaddr: The resulting socket address; should be a 2-tuple for @@ -363,3 +366,29 @@ def test_transportSemanticsToSocketType(self): self.assertEqual(socktypeU, SOCK_DGRAM) + def test_socketTypeToAddressType(self): + """ + When L{GAIResolver} receives a C{SOCK_DGRAM} result from + C{getaddrinfo}, it returns a C{'TCP'} L{IPv4Address} or L{IPv6Address}; + if it receives C{SOCK_STREAM} then it returns a C{'UDP'} type of same. + """ + receiver = ResultHolder(self) + flowInfo = 1 + scopeID = 2 + for socktype in SOCK_STREAM, SOCK_DGRAM: + self.getter.addResultForHost( + "example.com", ("::1", 0, flowInfo, scopeID), family=AF_INET6, + socktype=socktype + ) + self.getter.addResultForHost( + "example.com", ("127.0.0.3", 0), family=AF_INET, + socktype=socktype + ) + self.resolver.resolveHostName(receiver, u"example.com") + self.doThreadWork() + self.doReactorWork() + stream4, stream6, dgram4, dgram6 = receiver._addresses + self.assertEqual(stream4.type, 'TCP') + self.assertEqual(stream6.type, 'TCP') + self.assertEqual(dgram4.type, 'UDP') + self.assertEqual(dgram6.type, 'UDP') From 0c9d6cc416b8f7b8e94f7f93d6a9fccb6cc411a0 Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 23:51:06 -0800 Subject: [PATCH 28/34] compatibility layer --- src/twisted/internet/_resolver.py | 52 ++++++++++++ src/twisted/internet/test/test_resolver.py | 93 +++++++++++++++++++++- 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 4f493f8db23..fc5cc039f4c 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -18,8 +18,10 @@ from zope.interface import implementer from twisted.internet.interfaces import IHostnameResolver, IHostResolution +from twisted.internet.error import DNSLookupError from twisted.internet.threads import deferToThreadPool from twisted.internet.address import IPv4Address, IPv6Address +from twisted.logger import Logger @implementer(IHostResolution) @@ -127,3 +129,53 @@ def deliverResults(result): ) resolutionReceiver.resolutionComplete() return resolution + + + +@implementer(IHostnameResolver) +class SimpleResolverComplexifier(object): + """ + A converter from L{IResolverSimple} to L{IHostnameResolver}. + """ + + _log = Logger() + + def __init__(self, simpleResolver): + """ + Construct a L{SimpleResolverComplexifier} with an L{IResolverSimple}. + """ + self._simpleResolver = simpleResolver + + + def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, + addressTypes=None, transportSemantics='TCP'): + """ + See L{IHostnameResolver.resolveHostName} + + @param resolutionReceiver: see interface + + @param hostName: see interface + + @param portNumber: see interface + + @param addressTypes: see interface + + @param transportSemantics: see interface + + @return: see interface + """ + resolution = HostResolution(hostName) + resolutionReceiver.resolutionBegan(resolution) + onAddress = self._simpleResolver.getHostByName(hostName) + def addressReceived(address): + resolutionReceiver.addressResolved(IPv4Address('TCP', address, 0)) + def errorReceived(error): + if not error.check(DNSLookupError): + self._log.failure("while looking up {name} with {resolver}", + error, name=hostName, + resolver=self._simpleResolver) + onAddress.addCallbacks(addressReceived, errorReceived) + def finish(result): + resolutionReceiver.resolutionComplete() + onAddress.addCallback(finish) + return resolution diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index b81ed5ec221..a4fb66899f0 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -20,7 +20,7 @@ from zope.interface import implementer -from twisted.internet.interfaces import IResolutionReceiver +from twisted.internet.interfaces import IResolutionReceiver, IResolverSimple from twisted.trial.unittest import ( SynchronousTestCase as UnitTest @@ -30,7 +30,10 @@ from twisted._threads import createMemoryWorker, Team, LockWorker from twisted.internet.address import IPv4Address, IPv6Address -from twisted.internet._resolver import GAIResolver +from twisted.internet._resolver import GAIResolver, SimpleResolverComplexifier + +from twisted.internet.defer import Deferred +from twisted.internet.error import DNSLookupError class DeterministicThreadPool(ThreadPool, object): @@ -392,3 +395,89 @@ def test_socketTypeToAddressType(self): self.assertEqual(stream6.type, 'TCP') self.assertEqual(dgram4.type, 'UDP') self.assertEqual(dgram6.type, 'UDP') + + + +@implementer(IResolverSimple) +class SillyResolverSimple(object): + """ + Trivial implementation of L{IResolverSimple} + """ + def __init__(self): + """ + + """ + self._requests = [] + + + def getHostByName(self, name, timeout=()): + """ + Implement L{IResolverSimple.getHostByName} + """ + self._requests.append(Deferred()) + return self._requests[-1] + + + +class LegacyCompatibilityTests(UnitTest, object): + """ + Older applications may supply an object to the reactor via + C{installResolver} that only provides L{IResolverSimple}. + L{SimpleResolverComplexifier} is a wrapper for an L{IResolverSimple}. + """ + + def test_success(self): + """ + L{SimpleResolverComplexifier} translates C{resolveHostName} into + L{IResolutionReceiver.addressResolved}. + """ + simple = SillyResolverSimple() + complex = SimpleResolverComplexifier(simple) + receiver = ResultHolder(self) + self.assertEqual(receiver._started, False) + complex.resolveHostName(receiver, u"example.com") + self.assertEqual(receiver._started, True) + self.assertEqual(receiver._ended, False) + self.assertEqual(receiver._addresses, []) + simple._requests[0].callback("192.168.1.1") + self.assertEqual(receiver._addresses, + [IPv4Address('TCP', '192.168.1.1', 0)]) + self.assertEqual(receiver._ended, True) + + + def test_failure(self): + """ + L{SimpleResolverComplexifier} translates a known error result from + L{IResolverSimple.resolveHostName} into an empty result. + """ + simple = SillyResolverSimple() + complex = SimpleResolverComplexifier(simple) + receiver = ResultHolder(self) + self.assertEqual(receiver._started, False) + complex.resolveHostName(receiver, u"example.com") + self.assertEqual(receiver._started, True) + self.assertEqual(receiver._ended, False) + self.assertEqual(receiver._addresses, []) + simple._requests[0].errback(DNSLookupError("nope")) + self.assertEqual(receiver._ended, True) + self.assertEqual(receiver._addresses, []) + + + def test_error(self): + """ + L{SimpleResolverComplexifier} translates an unknown error result from + L{IResolverSimple.resolveHostName} into an empty result and a logged + error. + """ + simple = SillyResolverSimple() + complex = SimpleResolverComplexifier(simple) + receiver = ResultHolder(self) + self.assertEqual(receiver._started, False) + complex.resolveHostName(receiver, u"example.com") + self.assertEqual(receiver._started, True) + self.assertEqual(receiver._ended, False) + self.assertEqual(receiver._addresses, []) + simple._requests[0].errback(ZeroDivisionError("zow")) + self.assertEqual(len(self.flushLoggedErrors(ZeroDivisionError)), 1) + self.assertEqual(receiver._ended, True) + self.assertEqual(receiver._addresses, []) From 9094566c82b8a9b6183ca6a90bd2f46f567ed55a Mon Sep 17 00:00:00 2001 From: Glyph Date: Thu, 17 Nov 2016 23:56:05 -0800 Subject: [PATCH 29/34] distinct names since these need to live on the same object --- src/twisted/internet/interfaces.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/twisted/internet/interfaces.py b/src/twisted/internet/interfaces.py index 705ece2422f..1fa6488985f 100644 --- a/src/twisted/internet/interfaces.py +++ b/src/twisted/internet/interfaces.py @@ -1547,17 +1547,18 @@ class IReactorPluggableNameResolver(Interface): set to a user-supplied object. """ - resolver = Attribute( + nameResolver = Attribute( """ Read-only attribute; the resolver installed with L{installResolver}. + An L{IHostnameResolver}. """ ) - def installResolver(resolver): + def installNameResolver(resolver): """ Set the internal resolver to use for name lookups. - @type resolver: An object implementing the L{IResolverSimple} interface + @type resolver: An object providing the L{IHostnameResolver} interface. @param resolver: The new resolver to use. @return: The previously installed resolver. From 9cdc20bebf141ee43030ec4eec9af2a8cbfb8921 Mon Sep 17 00:00:00 2001 From: Glyph Date: Fri, 18 Nov 2016 00:04:32 -0800 Subject: [PATCH 30/34] twistedchecker --- src/twisted/internet/test/test_resolver.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index a4fb66899f0..c25fbf25074 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -405,14 +405,21 @@ class SillyResolverSimple(object): """ def __init__(self): """ - + Create a L{SillyResolverSimple} with a queue of requests it is working + on. """ self._requests = [] def getHostByName(self, name, timeout=()): """ - Implement L{IResolverSimple.getHostByName} + Implement L{IResolverSimple.getHostByName}. + + @param name: see L{IResolverSimple.getHostByName}. + + @param timeout: see L{IResolverSimple.getHostByName}. + + @return: see L{IResolverSimple.getHostByName}. """ self._requests.append(Deferred()) return self._requests[-1] From 06afa1e904fb90c92db9e36f0dbb9c4d102e0904 Mon Sep 17 00:00:00 2001 From: Glyph Date: Fri, 18 Nov 2016 01:12:35 -0800 Subject: [PATCH 31/34] forward and backward compatibility --- src/twisted/internet/_resolver.py | 77 +++++++++++++++++++++- src/twisted/internet/base.py | 41 +++++++++++- src/twisted/internet/test/test_resolver.py | 77 +++++++++++++++++++++- 3 files changed, 190 insertions(+), 5 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index fc5cc039f4c..5601f0acc2a 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -17,8 +17,10 @@ from zope.interface import implementer -from twisted.internet.interfaces import IHostnameResolver, IHostResolution +from twisted.internet.interfaces import (IHostnameResolver, IHostResolution, + IResolverSimple, IResolutionReceiver) from twisted.internet.error import DNSLookupError +from twisted.internet.defer import Deferred from twisted.internet.threads import deferToThreadPool from twisted.internet.address import IPv4Address, IPv6Address from twisted.logger import Logger @@ -179,3 +181,76 @@ def finish(result): resolutionReceiver.resolutionComplete() onAddress.addCallback(finish) return resolution + + + +@implementer(IResolutionReceiver) +class FirstOneWins(object): + """ + An L{IResolutionReceiver} which fires a L{Deferred} with its first result. + """ + + def __init__(self, deferred): + """ + @param deferred: The L{Deferred} to fire when the first resolution + result arrives. + """ + self._deferred = deferred + self._resolved = False + + + def resolutionBegan(self, resolution): + """ + + """ + self._resolution = resolution + + + def addressResolved(self, address): + """ + + """ + if self._resolved: + return + self._resolved = True + self._deferred.callback(address.host) + + + def resolutionComplete(self): + """ + + """ + if self._resolved: + return + self._deferred.errback(DNSLookupError(self._resolution.name)) + + + +@implementer(IResolverSimple) +class ComplexResolverSimplifier(object): + """ + A converter from L{IHostnameResolver} to L{IResolverSimple} + """ + def __init__(self, nameResolver): + """ + Create a L{ComplexResolverSimplifier} with an L{IHostnameResolver}. + + @param nameResolver: The L{IHostnameResolver} to use. + """ + self._nameResolver = nameResolver + + + def getHostByName(self, name, timeouts=()): + """ + See L{IResolverSimple.getHostByName} + + @param name: see L{IResolverSimple.getHostByName} + + @param timeouts: see L{IResolverSimple.getHostByName} + + @return: see L{IResolverSimple.getHostByName} + """ + result = Deferred() + self._nameResolver.resolveHostName(FirstOneWins(result), name, 0, + [IPv4Address]) + return result diff --git a/src/twisted/internet/base.py b/src/twisted/internet/base.py index 09931aec09f..728a7b16428 100644 --- a/src/twisted/internet/base.py +++ b/src/twisted/internet/base.py @@ -21,6 +21,11 @@ from twisted.internet.interfaces import IResolverSimple, IReactorPluggableResolver from twisted.internet.interfaces import IConnector, IDelayedCall from twisted.internet import fdesc, main, error, abstract, defer, threads +from twisted.internet._resolver import ( + GAIResolver as _GAIResolver, + ComplexResolverSimplifier as _ComplexResolverSimplifier, + SimpleResolverComplexifier as _SimpleResolverComplexifier, +) from twisted.python import log, failure, reflect from twisted.python.compat import unicode, iteritems from twisted.python.runtime import seconds as runtimeSeconds, platform @@ -487,6 +492,7 @@ def __init__(self): self._startedBefore = False # reactor internal readers, e.g. the waker. self._internalReaders = set() + self._nameResolver = None self.waker = None # Arrange for the running attribute to change to True at the right time @@ -509,12 +515,45 @@ def installWaker(self): raise NotImplementedError( reflect.qual(self.__class__) + " did not implement installWaker") + def installResolver(self, resolver): + """ + See L{IReactorPluggableResolver}. + + @param resolver: see L{IReactorPluggableResolver}. + + @return: see L{IReactorPluggableResolver}. + """ assert IResolverSimple.providedBy(resolver) oldResolver = self.resolver self.resolver = resolver + self._nameResolver = _SimpleResolverComplexifier(resolver) return oldResolver + + def installNameResolver(self, resolver): + """ + See L{IReactorPluggableNameResolver}. + + @param resolver: See L{IReactorPluggableNameResolver}. + + @return: see L{IReactorPluggableNameResolver}. + """ + previousNameResolver = self._nameResolver + self._nameResolver = resolver + self.resolver = _ComplexResolverSimplifier(resolver) + return previousNameResolver + + + @property + def nameResolver(self): + """ + Implementation of read-only + L{IReactorPluggableNameResolver.nameResolver}. + """ + return self._nameResolver + + def wakeUp(self): """ Wake up the event loop. @@ -939,7 +978,7 @@ def argChecker(arg): def _initThreads(self): self.usingThreads = True - self.resolver = ThreadedResolver(self) + self.installNameResolver(_GAIResolver(self, self.getThreadPool())) def callFromThread(self, f, *args, **kw): """ diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index c25fbf25074..4ab15d2ced9 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -13,12 +13,13 @@ from collections import defaultdict from socket import ( - gaierror, EAI_NONAME, AF_INET, AF_INET6, AF_UNSPEC, SOCK_STREAM, - SOCK_DGRAM, IPPROTO_TCP + getaddrinfo, gaierror, EAI_NONAME, AF_INET, AF_INET6, AF_UNSPEC, + SOCK_STREAM, SOCK_DGRAM, IPPROTO_TCP ) from threading import local, Lock from zope.interface import implementer +from zope.interface.verify import verifyObject from twisted.internet.interfaces import IResolutionReceiver, IResolverSimple @@ -30,10 +31,13 @@ from twisted._threads import createMemoryWorker, Team, LockWorker from twisted.internet.address import IPv4Address, IPv6Address -from twisted.internet._resolver import GAIResolver, SimpleResolverComplexifier +from twisted.internet._resolver import ( + GAIResolver, SimpleResolverComplexifier, ComplexResolverSimplifier +) from twisted.internet.defer import Deferred from twisted.internet.error import DNSLookupError +from twisted.internet.base import ReactorBase class DeterministicThreadPool(ThreadPool, object): @@ -488,3 +492,70 @@ def test_error(self): self.assertEqual(len(self.flushLoggedErrors(ZeroDivisionError)), 1) self.assertEqual(receiver._ended, True) self.assertEqual(receiver._addresses, []) + + + def test_simplifier(self): + """ + L{ComplexResolverSimplifier} translates an L{IHostnameResolver} into an + L{IResolverSimple} for applications that still expect the old + interfaces to be in place. + """ + self.pool, self.doThreadWork = deterministicPool() + self.reactor, self.doReactorWork = deterministicReactorThreads() + self.getter = FakeAddrInfoGetter() + self.resolver = GAIResolver(self.reactor, self.pool, + self.getter.getaddrinfo) + simpleResolver = ComplexResolverSimplifier(self.resolver) + self.getter.addResultForHost('example.com', ('192.168.3.4', 4321)) + success = simpleResolver.getHostByName('example.com') + failure = simpleResolver.getHostByName('nx.example.com') + self.doThreadWork() + self.doReactorWork() + self.doThreadWork() + self.doReactorWork() + self.assertEqual(self.failureResultOf(failure).type, DNSLookupError) + self.assertEqual(self.successResultOf(success), '192.168.3.4') + + + +class JustEnoughReactor(ReactorBase, object): + """ + Just enough subclass implementation to be a valid L{ReactorBase} subclass. + """ + def installWaker(self): + """ + Do nothing. + """ + + + +class ReactorInstallationTests(UnitTest, object): + """ + Tests for installing old and new resolvers onto a L{ReactorBase} (from + which all of Twisted's reactor implementations derive). + """ + + def test_defaultToGAIResolver(self): + """ + L{ReactorBase} defaults to using a L{GAIResolver}. + """ + reactor = JustEnoughReactor() + self.assertIsInstance(reactor.nameResolver, GAIResolver) + self.assertIs(reactor.nameResolver._getaddrinfo, getaddrinfo) + self.assertIsInstance(reactor.resolver, ComplexResolverSimplifier) + self.assertIs(reactor.nameResolver._reactor, reactor) + self.assertIs(reactor.nameResolver._threadpool, + reactor.getThreadPool()) + self.assertIs(reactor.resolver._nameResolver, reactor.nameResolver) + + + def test_installingOldStyleResolver(self): + """ + L{ReactorBase} will wrap an L{IResolverSimple} in a complexifier. + """ + reactor = JustEnoughReactor() + it = SillyResolverSimple() + verifyObject(IResolverSimple, reactor.installResolver(it)) + self.assertIsInstance(reactor.nameResolver, SimpleResolverComplexifier) + self.assertIs(reactor.nameResolver._simpleResolver, it) + From 7555e097a367bbe56c8c336017a21ccbad4b2d0a Mon Sep 17 00:00:00 2001 From: Glyph Date: Fri, 18 Nov 2016 01:51:56 -0800 Subject: [PATCH 32/34] defer acquisition of the threadpool acquiesce to the vagaries of reactor startup --- src/twisted/internet/_resolver.py | 17 ++++++++++------- src/twisted/internet/base.py | 3 ++- src/twisted/internet/test/test_resolver.py | 6 ++---- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 5601f0acc2a..375186c9207 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -72,24 +72,26 @@ class GAIResolver(object): L{getaddrinfo} in a thread. """ - def __init__(self, reactor, threadpool=None, getaddrinfo=getaddrinfo): + def __init__(self, reactor, getThreadPool=None, getaddrinfo=getaddrinfo): """ Create a L{GAIResolver}. @param reactor: the reactor to schedule result-delivery on @type reactor: L{IReactorThreads} - @param threadpool: the thread pool to use for scheduling name - resolutions. If not supplied, the use the given C{reactor}'s - thread pool. - @type threadpool: L{twisted.internet.threads} + @param getThreadPool: a function to retrieve the thread pool to use for + scheduling name resolutions. If not supplied, the use the given + C{reactor}'s thread pool. + @type getThreadPool: 0-argument callable returning a + L{twisted.python.threadpool.ThreadPool} @param getaddrinfo: a reference to the L{getaddrinfo} to use - mainly parameterized for testing. @type getaddrinfo: callable with the same signature as L{getaddrinfo} """ self._reactor = reactor - self._threadpool = threadpool + self._getThreadPool = (reactor.getThreadPool if getThreadPool is None + else getThreadPool) self._getaddrinfo = getaddrinfo @@ -110,6 +112,7 @@ def resolveHostName(self, resolutionReceiver, hostName, portNumber=0, @return: see interface """ + pool = self._getThreadPool() addressFamily = _typesToAF[_any if addressTypes is None else frozenset(addressTypes)] socketType = _transportToSocket[transportSemantics] @@ -119,7 +122,7 @@ def get(): socketType) except gaierror: return [] - d = deferToThreadPool(self._reactor, self._threadpool, get) + d = deferToThreadPool(self._reactor, pool, get) resolution = HostResolution(hostName) resolutionReceiver.resolutionBegan(resolution) @d.addCallback diff --git a/src/twisted/internet/base.py b/src/twisted/internet/base.py index 728a7b16428..7a99a0f1c8c 100644 --- a/src/twisted/internet/base.py +++ b/src/twisted/internet/base.py @@ -977,8 +977,9 @@ def argChecker(arg): threadpoolShutdownID = None def _initThreads(self): + self.installNameResolver(_GAIResolver(self, self.getThreadPool)) self.usingThreads = True - self.installNameResolver(_GAIResolver(self, self.getThreadPool())) + def callFromThread(self, f, *args, **kw): """ diff --git a/src/twisted/internet/test/test_resolver.py b/src/twisted/internet/test/test_resolver.py index 4ab15d2ced9..a08c67eaf3f 100644 --- a/src/twisted/internet/test/test_resolver.py +++ b/src/twisted/internet/test/test_resolver.py @@ -238,7 +238,7 @@ def setUp(self): self.pool, self.doThreadWork = deterministicPool() self.reactor, self.doReactorWork = deterministicReactorThreads() self.getter = FakeAddrInfoGetter() - self.resolver = GAIResolver(self.reactor, self.pool, + self.resolver = GAIResolver(self.reactor, lambda: self.pool, self.getter.getaddrinfo) @@ -503,7 +503,7 @@ def test_simplifier(self): self.pool, self.doThreadWork = deterministicPool() self.reactor, self.doReactorWork = deterministicReactorThreads() self.getter = FakeAddrInfoGetter() - self.resolver = GAIResolver(self.reactor, self.pool, + self.resolver = GAIResolver(self.reactor, lambda: self.pool, self.getter.getaddrinfo) simpleResolver = ComplexResolverSimplifier(self.resolver) self.getter.addResultForHost('example.com', ('192.168.3.4', 4321)) @@ -544,8 +544,6 @@ def test_defaultToGAIResolver(self): self.assertIs(reactor.nameResolver._getaddrinfo, getaddrinfo) self.assertIsInstance(reactor.resolver, ComplexResolverSimplifier) self.assertIs(reactor.nameResolver._reactor, reactor) - self.assertIs(reactor.nameResolver._threadpool, - reactor.getThreadPool()) self.assertIs(reactor.resolver._nameResolver, reactor.nameResolver) From 65eebe92403bc47fdb75c0148b46a85685a02b7e Mon Sep 17 00:00:00 2001 From: Glyph Date: Fri, 18 Nov 2016 03:04:36 -0800 Subject: [PATCH 33/34] twistedchecker --- src/twisted/internet/_resolver.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/twisted/internet/_resolver.py b/src/twisted/internet/_resolver.py index 375186c9207..fc68d958edc 100644 --- a/src/twisted/internet/_resolver.py +++ b/src/twisted/internet/_resolver.py @@ -204,14 +204,18 @@ def __init__(self, deferred): def resolutionBegan(self, resolution): """ - + See L{IResolutionReceiver.resolutionBegan} + + @param resolution: See L{IResolutionReceiver.resolutionBegan} """ self._resolution = resolution def addressResolved(self, address): """ - + See L{IResolutionReceiver.addressResolved} + + @param address: See L{IResolutionReceiver.addressResolved} """ if self._resolved: return @@ -221,7 +225,7 @@ def addressResolved(self, address): def resolutionComplete(self): """ - + See L{IResolutionReceiver.resolutionComplete} """ if self._resolved: return From 6625d4b725d1b4f6a89e2839d9f119bb12980f8f Mon Sep 17 00:00:00 2001 From: Glyph Date: Sat, 26 Nov 2016 02:09:10 -0500 Subject: [PATCH 34/34] too many newlines as per review --- src/twisted/internet/interfaces.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/twisted/internet/interfaces.py b/src/twisted/internet/interfaces.py index 1fa6488985f..62e8bfcfc3a 100644 --- a/src/twisted/internet/interfaces.py +++ b/src/twisted/internet/interfaces.py @@ -188,7 +188,6 @@ def resolveHostName(resolutionReceiver, hostName, portNumber=0, - class IResolver(IResolverSimple): def query(query, timeout=None): """