Skip to content
Permalink
Browse files

Add User-Agent header when resolving via urllib (#663)

At least some Python Index servers return a simple format of packages page, redirecting to / to /+simple/.

So there is a different result between `pex` run using `urllib` and using `requests` - in one case it errors.
  • Loading branch information...
AndrewLvov authored and jsirois committed Feb 6, 2019
1 parent 7dabca5 commit fe405f9965dd1698095dde8241c07f8ef2a41517
Showing with 4 additions and 3 deletions.
  1. +4 −3 pex/http.py
@@ -57,6 +57,7 @@ class Context(AbstractClass):
"""

DEFAULT_ENCODING = 'iso-8859-1'
USER_AGENT = 'pex/%s' % PEX_VERSION

class Error(Exception):
"""Error base class for Contexts to wrap application-specific exceptions."""
@@ -135,7 +136,8 @@ class UrllibContext(Context):
"""Default Python standard library Context."""

def open(self, link):
return urllib_request.urlopen(link.url)
request = urllib_request.Request(link.url, headers={'User-Agent': self.USER_AGENT})
return urllib_request.urlopen(request)

def content(self, link):
if link.local:
@@ -146,7 +148,7 @@ def content(self, link):
return fp.read().decode(encoding, 'replace')

def resolve(self, link):
request = urllib_request.Request(link.url)
request = urllib_request.Request(link.url, headers={'User-Agent': self.USER_AGENT})
request.get_method = lambda: 'HEAD'
with contextlib.closing(urllib_request.urlopen(request)) as response:
return link.wrap(response.url)
@@ -209,7 +211,6 @@ def close(self):

class RequestsContext(Context):
"""A requests-based Context."""
USER_AGENT = 'pex/%s' % PEX_VERSION

@staticmethod
def _create_session(max_retries):

0 comments on commit fe405f9

Please sign in to comment.
You can’t perform that action at this time.