Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

allow explicitely disabling hubproxy for request

  • Loading branch information...
commit 57afe963e0834a6a88cf3da77beb6ad5cdc4ccf3 1 parent 023d642
@alexcepoi authored
View
4 .gitignore
@@ -0,0 +1,4 @@
+# temp files
+**swp
+**pyc
+**~
View
2  scrapylib/hubproxy.py
@@ -41,7 +41,7 @@ def get_proxyauth(self, spider):
return basic_auth_header(self.user, getattr(self, 'pass'))
def process_request(self, request, spider):
- if self.enabled:
+ if self.enabled and 'dont_proxy' not in request.meta:
request.meta['proxy'] = self.url
request.meta['download_timeout'] = self.download_timeout
request.headers['Proxy-Authorization'] = self._proxyauth
View
10 scrapylib/tests/test_hubproxy.py
@@ -57,6 +57,16 @@ def _assert_enabled(self, spider,
res = Response(req.url)
assert mw.process_response(req, res, spider) is res
+ # disabled if 'dont_proxy' is set
+ req = Request('http://www.scrapytest.org')
+ req.meta['dont_proxy'] = True
+ assert mw.process_request(req, spider) is None
+ self.assertEqual(req.meta.get('proxy'), None)
+ self.assertEqual(req.meta.get('download_timeout'), None)
+ self.assertEqual(req.headers.get('Proxy-Authorization'), None)
+ res = Response(req.url)
+ assert mw.process_response(req, res, spider) is res
+
if maxbans > 0:
# assert ban count is reseted after a succesful response
res = Response('http://ban.me', status=bancode)
Please sign in to comment.
Something went wrong with that request. Please try again.