Skip to content


Subversion checkout URL

You can clone with
Download ZIP
Tree: 823cf3d7dd
Fetching contributors…

Cannot retrieve contributors at this time

25 lines (17 sloc) 651 Bytes
import socket
from twisted.internet import defer
from twisted.internet.base import ThreadedResolver
from scrapy.utils.datatypes import LocalCache
# TODO: cache misses
# TODO: make cache size a setting
dnscache = LocalCache(10000)
class CachingThreadedResolver(ThreadedResolver):
def getHostByName(self, name, timeout = (1, 3, 11, 45)):
if name in dnscache:
return defer.succeed(dnscache[name])
d = ThreadedResolver.getHostByName(self, name, timeout)
d.addCallback(self._cache_result, name)
return d
def _cache_result(self, result, name):
dnscache[name] = result
return result
Jump to Line
Something went wrong with that request. Please try again.