HTTPS proxy not being used - max retries exceeded #1218

gfairchild · 2013-02-26T17:06:10Z

I'm behind a proxy at work, and I'm having issues getting requests to use the proxy. urllib2 is able to use it just fine, but requests fails. I've tried both setting an environment variable (both HTTPS_PROXY and https_proxy) and passing in a dict, but neither work.

I'm on OSX 10.7.5 using Python 2.7.3 and requests 1.1.0 installed in a virtualenv via pip.

(osx)gfairchild@stueyemac ~> set | grep -i proxy
HTTPS_PROXY=https://proxy.com:8080
https_proxy=https://proxy.com:8080
(osx)gfairchild@stueyemac ~> python
Python 2.7.3 (v2.7.3:70274d53c1dd, Apr  9 2012, 20:52:43) 
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import urllib2
>>> r = urllib2.urlopen('https://google.com')
>>> print r.read()
<!doctype html><html itemscope="itemscope" itemtype="http://schema.org/WebPage"><head><meta content="Search the world's information, including webpages, images, videos and more. Google has many special features to help you find exactly what you're looking for." name="description"><meta content="noodp" name="robots"><meta itemprop="image" content="/images/google_favicon_128.png"><title>Google</title><script>(function(){
window.google={kEI:"I-gsUdvaLMn-0gHWhIHIAg",getEI:function(a){for(var b;a&&(!a.getAttribute||!(b=a.getAttribute("eid")));)a=a.parentNode;return b||google.kEI},https:function(){return"https:"==window.location.protocol},kEXPI:"17259,18168,39523,4000116,4001569,4001948,4001959,4001975,4002001,4002159,4002562,4002734,4002855,4002858,4003372,4003374,4003387,4003514,4003575,4003638,4003917,4003944,4003982,4004015,4004064,4004074,4004083,4004152,4004181,4004214,4004241,4004276,4004298",kCSI:{e:"17259,18168,39523,4000116,4001569,4001948,4001959,4001975,4002001,4002159,4002562,4002734,4002855,4002858,4003372,4003374,4003387,4003514,4003575,4003638,4003917,4003944,4003982,4004015,4004064,4004074,4004083,4004152,4004181,4004214,4004241,4004276,4004298",ei:"I-gsUdvaLMn-0gHWhIHIAg"},authuser:0,ml:function(){},kHL:"en",time:function(){return(new Date).getTime()},log:function(a,
b,c,k){var d=new Image,f=google.lc,e=google.li,g="";d.onerror=d.onload=d.onabort=function(){delete f[e]};f[e]=d;!c&&-1==b.search("&ei=")&&(g="&ei="+google.getEI(k));c=c||"/gen_204?atyp=i&ct="+a+"&cad="+b+g+"&zx="+google.time();a=/^http:/i;a.test(c)&&google.https()?(google.ml(Error("GLMM"),!1,{src:c}),delete f[e]):(d.src=c,google.li=e+1)},lc:[],li:0,Toolbelt:{},y:{},x:function(a,b){google.y[a.id]=[a,b];return!1},load:function(a,b){google.x({id:"l"+a},function(){google.load(a,b)})}};
})();
(function(){var d=!1;google.sn="webhp";google.timers={};google.startTick=function(a,b){google.timers[a]={t:{start:google.time()},bfr:!!b}};google.tick=function(a,b,h){google.timers[a]||google.startTick(a);google.timers[a].t[b]=h||google.time()};google.startTick("load",!0);
try{}catch(e){}})();
var _gjwl=location;function _gjuc(){var a=_gjwl.href.indexOf("#");if(0<=a&&(a=_gjwl.href.substring(a),0<a.indexOf("&q=")||0<=a.indexOf("#q=")))if(a=a.substring(1),-1==a.indexOf("#")){for(var d=0;d<a.length;){var b=d;"&"==a.charAt(b)&&++b;var c=a.indexOf("&",b);-1==c&&(c=a.length);b=a.substring(b,c);if(0==b.indexOf("fp="))a=a.substring(0,d)+a.substring(c,a.length),c=d;else if("cad=h"==b)return 0;d=c}_gjwl.href="/search?"+a+"&cad=h";return 1}return 0}
function _gjp(){(!window._gjwl.hash||!window._gjuc())&&setTimeout(_gjp,500)};
window._gjp&&_gjp();</script><style>#gbar,#guser{font-size:13px;padding-top:1px !important;}#gbar{height:22px}#guser{padding-bottom:7px !important;text-align:right}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}@media all{.gb1{height:22;margin-right:.5em;vertical-align:top}#gbar{float:left}}a.gb1,a.gb4{text-decoration:underline !important}a.gb1,a.gb4{color:#00c !important}.gbi .gb4{color:#dd8e27 !important}.gbf .gb4{color:#900 !important}</style><style>.h{font-family:arial,sans-serif}body{font-family:arial,sans-serif}td{font-family:arial,sans-serif}a{font-family:arial,sans-serif}p{font-family:arial,sans-serif}body{margin:0;overflow-y:scroll}#gog{padding:3px 8px 0}.h{color:#36c}.q{color:#00c}.ts{border-collapse:collapse}td{line-height:.8em}.gac_m td{line-height:17px}form{margin-bottom:20px}.ts td{padding:0}em{font-weight:bold;font-style:normal}.lst{height:25px;width:496px;font:18px arial,sans-serif}.gsfi{font:18px arial,sans-serif}.gsfs{font:17px arial,sans-serif}.ds{display:inline-box;display: inline-block;margin:3px 0 4px;margin-left:4px}input{font-family:inherit}body{background:#fff;color:black}a.gb1{color:#11c !important}a.gb2{color:#11c !important}a.gb3{color:#11c !important}a.gb4{color:#11c !important}.sblc{padding-top:5px}.lsbb{background:#eee;border:solid 1px;border-color:#ccc #999 #999 #ccc;height:30px}a{color:#11c;text-decoration:none}a:hover{text-decoration:underline}a:active{text-decoration:underline}.fl a{color:#36c}a:visited{color:#551a8b}a.gb1{text-decoration:underline}a.gb4{text-decoration:underline}a.gb3:hover{text-decoration:none}.sblc a{display:block;margin:2px 0;margin-left:13px;font-size:11px}#ghead a.gb2:hover{color:#fff !important}.lsbb{display:block}.ftl{display:inline-block;margin:0 12px}.lsb{background:url(/images/srpr/nav_logo80.png) 0 -258px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px arial,sans-serif;vertical-align:top}#fll a{display:inline-block;margin:0 12px}.lsb:active{background:#ccc}.lst:focus{outline:none}#addlang a{padding:0 3px}</style><script></script> </head><body dir="ltr" bgcolor="#fff"><script>(function(){var src='/images/srpr/nav_logo80.png';var iesg=false;document.body.onload = function(){window.n && window.n();if (document.images){new Image().src=src;}
if (!iesg){document.f&&document.f.q.focus();document.gbqf&&document.gbqf.q.focus();}
}
})();</script><textarea id="csi" style="display:none"></textarea><div id="mngb"><div id=gbar><nobr><b class=gb1>Search</b> <a class=gb1 href="https://www.google.com/imghp?hl=en&tab=wi">Images</a> <a class=gb1 href="https://maps.google.com/maps?hl=en&tab=wl">Maps</a> <a class=gb1 href="https://play.google.com/?hl=en&tab=w8">Play</a> <a class=gb1 href="https://www.youtube.com/?tab=w1">YouTube</a> <a class=gb1 href="https://news.google.com/nwshp?hl=en&tab=wn">News</a> <a class=gb1 href="https://mail.google.com/mail/?tab=wm">Gmail</a> <a class=gb1 href="https://drive.google.com/?tab=wo">Drive</a> <a class=gb1 style="text-decoration:none" href="http://www.google.com/intl/en/options/"><u>More</u> &raquo;</a></nobr></div><div id=guser width=100%><nobr><span id=gbn class=gbi></span><span id=gbf class=gbf></span><span id=gbe></span><a href="http://www.google.com/history/optout?hl=en" class=gb4>Web History</a> | <a  href="/preferences?hl=en" class=gb4>Settings</a> | <a target=_top id=gb_70 href="https://accounts.google.com/ServiceLogin?hl=en&continue=https://www.google.com/" class=gb4>Sign in</a></nobr></div><div class=gbh style=left:0></div><div class=gbh style=right:0></div></div><iframe name="wgjf" style="display:none"></iframe><center><br clear="all" id="lgpd"><div id="lga"><img alt="Google" height="95" src="/intl/en_ALL/images/srpr/logo1w.png" width="275" id="hplogo" onload="window.lol&&lol()" style="padding:28px 0 14px"><br><br></div><form action="/search" name="f"><table cellpadding="0" cellspacing="0"><tr valign="top"><td width="25%">&nbsp;</td><td align="center" nowrap="nowrap"><input name="ie" value="ISO-8859-1" type="hidden"><input value="en" name="hl" type="hidden"><input name="source" type="hidden" value="hp"><div class="ds" style="height:32px;margin:4px 0"><input autocomplete="off" class="lst" value="" title="Google Search" maxlength="2048" name="q" size="57" style="color:#000;margin:0;padding:5px 8px 0 6px;vertical-align:top"></div><br style="line-height:0"><span class="ds"><span class="lsbb"><input class="lsb" value="Google Search" name="btnG" type="submit"></span></span><span class="ds"><span class="lsbb"><input class="lsb" value="I'm Feeling Lucky" name="btnI" type="submit" onclick="if(this.form.q.value)this.checked=1; else top.location='/doodles/'"></span></span></td><td class="fl sblc" align="left" nowrap="nowrap" width="25%"><a href="/advanced_search?hl=en&amp;authuser=0">Advanced search</a><a href="/language_tools?hl=en&amp;authuser=0">Language tools</a></td></tr></table><input type="hidden" id="gbv" name="gbv" value="1"></form><div id="gac_scont"></div><div style="font-size:83%;min-height:3.5em"><br></div><span id="footer"><div style="font-size:10pt"><div id="fll" style="margin:19px auto;text-align:center"><a href="/intl/en/ads/">Advertising&nbsp;Programs</a><a href="/services/">Business Solutions</a><a href="https://plus.google.com/116899029375914044550" rel="publisher">+Google</a><a href="/intl/en/about.html">About Google</a></div></div><p style="color:#767676;font-size:8pt">&copy; 2012 - <a href="/intl/en/policies/">Privacy & Terms</a></p></span></center><div id=xjsd></div><div id=xjsi><script>if(google.y)google.y.first=[];(function(){var b;function c(a){window.setTimeout(function(){var d=document.createElement("script");d.src=a;document.getElementById("xjsd").appendChild(d)},0)}google.dljp=function(a){b=a;google.xjsi||(google.xjsu=a,c(b))};google.dlj=c;})();
if(!google.xjs){google.dstr=[];google.rein=[];window._=window._||{};window._._DumpException=function(e){throw e};if(google.timers&&google.timers.load.t){google.timers.load.t.xjsls=new Date().getTime();}google.dljp('/xjs/_/js/hp/sb_he,pcc/rt\x3dj/ver\x3ddUr8sRKOHrQ.en_US./d\x3d1/sv\x3d1/rs\x3dAItRSTMOuVd64TXf59sLZpeyfeaOnqHKgQ');google.xjs=1;}google.pmc={sb:{"agen":false,"cgen":true,"client":"heirloom-hp","dh":true,"ds":"","eqch":true,"fl":true,"host":"google.com","jsonp":true,"msgs":{"lcky":"I\u0026#39;m Feeling Lucky","lml":"Learn more","oskt":"Input tools","psrc":"This search was removed from your \u003Ca href=\"/history\"\u003EWeb History\u003C/a\u003E","psrl":"Remove","sbit":"Search by image","srch":"Google Search"},"ovr":{"l":1,"ms":1},"pq":"","qcpw":false,"scd":10,"sce":5,"stok":"2TLNAUa1t9yVii-aHCy9IeO1J4M"},hp:{},pcc:{}};google.y.first.push(function(){if(google.med){google.med('init');google.initHistory();google.med('history');}google.History&&google.History.initialize('/');google.hs&&google.hs.init&&google.hs.init()});if(google.j&&google.j.en&&google.j.xi){window.setTimeout(google.j.xi,0);}</script></div><script>(function(){var b,c,d,e;function g(a,f){a.removeEventListener?(a.removeEventListener("load",f,!1),a.removeEventListener("error",f,!1)):(a.detachEvent("onload",f),a.detachEvent("onerror",f))}function h(a){e=(new Date).getTime();++c;a=a||window.event;a=a.target||a.srcElement;g(a,h)}var k=document.getElementsByTagName("img");b=k.length;
for(var l=c=0,m;l<b;++l)m=k[l],m.complete||"string"!=typeof m.src||!m.src?++c:m.addEventListener?(m.addEventListener("load",h,!1),m.addEventListener("error",h,!1)):(m.attachEvent("onload",h),m.attachEvent("onerror",h));d=b-c;
function n(){if(google.timers.load.t){google.timers.load.t.ol=(new Date).getTime();google.timers.load.t.iml=e;google.kCSI.imc=c;google.kCSI.imn=b;google.kCSI.imp=d;void 0!==google.stt&&(google.kCSI.stt=google.stt);google.csiReport&&google.csiReport()}}window.addEventListener?window.addEventListener("load",n,!1):window.attachEvent&&window.attachEvent("onload",n);google.timers.load.t.prt=e=(new Date).getTime();})();
</script></body></html>
>>> import requests
>>> r = requests.get('https://google.com')
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/api.py", line 55, in get
    return request('get', url, **kwargs)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/api.py", line 44, in request
    return session.request(method=method, url=url, **kwargs)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/sessions.py", line 279, in request
    resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/sessions.py", line 374, in send
    r = adapter.send(request, **kwargs)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/adapters.py", line 209, in send
    raise ConnectionError(e)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='proxy.com', port=8080): Max retries exceeded with url: https://google.com/ (Caused by <class 'socket.error'>: [Errno 54] Connection reset by peer)
>>> p = {'https': 'https://proxy.com:8080',}
>>> r = requests.get('https://google.com', proxies=p)
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/api.py", line 55, in get
    return request('get', url, **kwargs)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/api.py", line 44, in request
    return session.request(method=method, url=url, **kwargs)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/sessions.py", line 279, in request
    resp = self.send(prep, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/sessions.py", line 374, in send
    r = adapter.send(request, **kwargs)
  File "/Users/gfairchild/Documents/work/compepi/Twitter/lib/osx/lib/python2.7/site-packages/requests/adapters.py", line 209, in send
    raise ConnectionError(e)
requests.exceptions.ConnectionError: HTTPSConnectionPool(host='proxy.com', port=8080): Max retries exceeded with url: https://google.com/ (Caused by <class 'socket.error'>: [Errno 54] Connection reset by peer)

The text was updated successfully, but these errors were encountered:

sigmavirus24 · 2013-02-26T17:13:07Z

I believe this is a duplicate and related to urllib3/urllib3#139

@Lukasa thoughts?

gfairchild · 2013-02-26T17:30:22Z

Ah yes, it looks like you may be right. I just looked at the open issues and didn't think to look at the closed issues.

sigmavirus24 · 2013-02-26T17:35:29Z

No worries. We're just waiting for that pull request to land before we can support this. I believe @t-8ch has made the case that you can use http:// for your proxy and it should work, but I've no practical knowledge of how proxies work in all candor.

sigmavirus24 · 2013-02-26T17:37:41Z

Closing as a duplicate (#1182)

gfairchild · 2013-02-26T17:39:39Z

I tried using http instead, but what I get is an internal error back:

(osx)gfairchild@stueyemac ~> python
Python 2.7.3 (v2.7.3:70274d53c1dd, Apr  9 2012, 20:52:43) 
[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] on darwin
Type "help", "copyright", "credits" or "license" for more information.
>>> import requests
>>> p = {'https': 'http://proxy.com:8080',}
>>> r = requests.get('https://google.com', proxies=p)
>>> r.text
u'<HTML><HEAD>\n<TITLE>Request Error</TITLE>\n</HEAD>\n<BODY>\n<FONT face="Helvetica">\n<big><strong></strong></big><BR>\n</FONT>\n<blockquote>\n<TABLE border=0 cellPadding=1 width="80%">\n<TR><TD>\n<FONT face="Helvetica">\n<big>Request Error (invalid_request)</big>\n<BR>\n<BR>\n</FONT>\n</TD></TR>\n<TR><TD>\n<FONT face="Helvetica">\nYour request could not be processed. Request could not be handled\n</FONT>\n</TD></TR>\n<TR><TD>\n<FONT face="Helvetica">\nThis could be caused by a misconfiguration, or possibly a malformed request.\n</FONT>\n</TD></TR>\n<TR><TD>\n<FONT face="Helvetica" SIZE=2>\n<BR>\nFor assistance, contact your network support team.\n</FONT>\n</TD></TR>\n</TABLE>\n</blockquote>\n</FONT>\n</BODY></HTML>\n'

Unfortunately, it looks like I'll have to either use urllib2 for now or wait for this to be fixed. Anyway, thanks for your help!

t-8ch · 2013-02-26T18:09:51Z

I'll recap the issue with the scheme :-)

Most people use one proxy for both http and https.
The connection between requests and the proxy is mostly http. (This is what the
scheme in the proxy url is for)
The proxy can handle plain http requests just fine.
Encrypted connections are just some random tcp stream to the proxy. This is
where the http 'CONNECT' verb is used. Http requests made using CONNECT are sent to the proxy
where the body of the request is the encoded raw data of the tcp connection.
The proxies then decodes the body and forwards the raw data, receives the
response, reencodes it and sends the raw, encoded response back to the requester.
(This is how all those http proxy tunnels work)
Urllib3 doesn't support CONNECT at the moment (will be supported very soon!).
Proxied https requests are therefore sent in plaintext in the form of
GET https://httpbin.org/ HTTP/1.1
This is not how https should be proxied and why you get the error at your
proxy. (My squid for example doesn't care and does the right, erm.. wrong thing)
(@gfairchild: this is the reason for your last error)

Again, the reason one should add the scheme is that you most will likely use the same
proxy for both http and https. As requests will guess the proxy scheme based
on the request scheme.
This is unfortunate as both aren't correlated.
This way you may end up connecting to your proxy on the same port/ip one time
with plain http and the other time with https.
Your proxy most likely won't support this.
This is the reason for all those "Unknown SSL version" bug reports. Request is
trying to establish a ssl connection with a non-ssl proxy which sends plain
http responses and therefore the handshake has no chance to succeed.

gfairchild · 2013-02-26T18:15:09Z

That makes perfect sense. Thanks for explaining it so succinctly! I look forward to urllib3's support of the CONNECT verb!

liword · 2018-02-06T22:53:46Z

same issue here.

sigmavirus24 closed this as completed Feb 26, 2013

Damgaard mentioned this issue Aug 16, 2013

Not using proxy correctly? praw-dev/praw#237

Closed

github-actions bot locked as resolved and limited conversation to collaborators Sep 8, 2021

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

HTTPS proxy not being used - max retries exceeded #1218

HTTPS proxy not being used - max retries exceeded #1218

gfairchild commented Feb 26, 2013

sigmavirus24 commented Feb 26, 2013

gfairchild commented Feb 26, 2013

sigmavirus24 commented Feb 26, 2013

sigmavirus24 commented Feb 26, 2013

gfairchild commented Feb 26, 2013

t-8ch commented Feb 26, 2013

gfairchild commented Feb 26, 2013

liword commented Feb 6, 2018

HTTPS proxy not being used - max retries exceeded #1218

HTTPS proxy not being used - max retries exceeded #1218

Comments

gfairchild commented Feb 26, 2013

sigmavirus24 commented Feb 26, 2013

gfairchild commented Feb 26, 2013

sigmavirus24 commented Feb 26, 2013

sigmavirus24 commented Feb 26, 2013

gfairchild commented Feb 26, 2013

t-8ch commented Feb 26, 2013

gfairchild commented Feb 26, 2013

liword commented Feb 6, 2018