Skip to content

Commit

Permalink
Fix downloads over socks5
Browse files Browse the repository at this point in the history
The aiohttp_socks library requires a separate flag to do DNS lookups
through the proxy, and failure to set this flag was leaking DNS
lookups outside of the SOCKS proxy.
  • Loading branch information
mehaase committed Jun 3, 2020
1 parent fa08c7e commit 92d6a1d
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion starbelly/downloader.py
Expand Up @@ -15,6 +15,7 @@
logger = logging.getLogger(__name__)
_HTTP_PROXY_SCHEMES = ('http', 'https')
_SOCKS_PROXY_SCHEMES = ('socks4', 'socks4a', 'socks5')
_RDNS_SOCKS_PROXY_SCHEMES = ('socks4a', 'socks5')


class CrawlItemLimitExceeded(Exception):
Expand Down Expand Up @@ -256,8 +257,9 @@ async def _download_asyncio(self, request, skip_mime=False):
proxy_type, proxy_url = self._policy.proxy_rules.get_proxy_url(url)

if proxy_type in _SOCKS_PROXY_SCHEMES:
rdns = proxy_type in _RDNS_SOCKS_PROXY_SCHEMES
session_args['connector'] = aiohttp_socks.SocksConnector.from_url(
proxy_url)
proxy_url, rdns=rdns)
else:
session_args['connector'] = aiohttp.TCPConnector(verify_ssl=False)

Expand Down

0 comments on commit 92d6a1d

Please sign in to comment.