Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

file 24 lines (17 sloc) 0.651 kb
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
import socket

from twisted.internet import defer
from twisted.internet.base import ThreadedResolver

from scrapy.utils.datatypes import LocalCache

# TODO: cache misses
# TODO: make cache size a setting

dnscache = LocalCache(10000)

class CachingThreadedResolver(ThreadedResolver):

    def getHostByName(self, name, timeout = (1, 3, 11, 45)):
        if name in dnscache:
            return defer.succeed(dnscache[name])
        d = ThreadedResolver.getHostByName(self, name, timeout)
        d.addCallback(self._cache_result, name)
        return d

    def _cache_result(self, result, name):
        dnscache[name] = result
        return result
Something went wrong with that request. Please try again.