Skip to content

Commit

Permalink
Add HTTPS proxy support
Browse files Browse the repository at this point in the history 
Squashed python-hyper#322

# Conflicts:
#	hyper/contrib.py
  • Loading branch information
@KostyaEsmukov
KostyaEsmukov authored and Camille Fabreguettes committed Jun 2, 2017
1 parent f422d00 commit 411c1c9
Show file tree
Hide file tree
Showing 13 changed files with 775 additions and 93 deletions.
11 changes: 7 additions & 4 deletions hyper/common/connection.py
View file
Expand Up @@ -44,8 +44,9 @@ class HTTPConnection(object):
:param proxy_host: (optional) The proxy to connect to. This can be an IP
address or a host name and may include a port.
:param proxy_port: (optional) The proxy port to connect to. If not provided
and one also isn't provided in the ``proxy`` parameter, defaults to
8080.
and one also isn't provided in the ``proxy_host`` parameter, defaults
to 8080.
:param proxy_headers: (optional) The headers to send to a proxy.
"""
def __init__(self,
host,
Expand All @@ -56,19 +57,21 @@ def __init__(self,
ssl_context=None,
proxy_host=None,
proxy_port=None,
proxy_headers=None,
**kwargs):

self._host = host
self._port = port
self._h1_kwargs = {
'secure': secure, 'ssl_context': ssl_context,
'proxy_host': proxy_host, 'proxy_port': proxy_port,
'enable_push': enable_push
'proxy_headers': proxy_headers, 'enable_push': enable_push
}
self._h2_kwargs = {
'window_manager': window_manager, 'enable_push': enable_push,
'secure': secure, 'ssl_context': ssl_context,
'proxy_host': proxy_host, 'proxy_port': proxy_port
'proxy_host': proxy_host, 'proxy_port': proxy_port,
'proxy_headers': proxy_headers
}

# Add any unexpected kwargs to both dictionaries.
Expand Down
19 changes: 19 additions & 0 deletions hyper/common/exceptions.py
View file
Expand Up @@ -71,3 +71,22 @@ class MissingCertFile(Exception):
The certificate file could not be found.
"""
pass


# Create our own ConnectionError.
try: # pragma: no cover
ConnectionError = ConnectionError
except NameError: # pragma: no cover
class ConnectionError(Exception):
"""
An error occurred during connection to a host.
"""


class ProxyError(ConnectionError):
"""
An error occurred during connection to a proxy.
"""
def __init__(self, message, response):
self.response = response
super(ProxyError, self).__init__(message)
41 changes: 33 additions & 8 deletions hyper/contrib.py
View file
Expand Up @@ -9,7 +9,9 @@
from requests.adapters import HTTPAdapter
from requests.models import Response
from requests.structures import CaseInsensitiveDict
from requests.utils import get_encoding_from_headers
from requests.utils import (
get_encoding_from_headers, select_proxy, prepend_scheme_if_needed
)
from requests.cookies import extract_cookies_to_jar
except ImportError: # pragma: no cover
HTTPAdapter = object
Expand All @@ -29,7 +31,8 @@ def __init__(self, *args, **kwargs):
#: A mapping between HTTP netlocs and ``HTTP20Connection`` objects.
self.connections = {}

def get_connection(self, host, port, scheme, cert=None, verify=True):
def get_connection(self, host, port, scheme, cert=None, verify=True,
proxy=None):
"""
Gets an appropriate HTTP/2 connection object based on
host/port/scheme/cert tuples.
Expand All @@ -50,29 +53,51 @@ def get_connection(self, host, port, scheme, cert=None, verify=True):
elif verify is not True:
ssl_context = init_context(cert_path=verify, cert=cert)

if proxy:
proxy_headers = self.proxy_headers(proxy)
proxy_netloc = urlparse(proxy).netloc
else:
proxy_headers = None
proxy_netloc = None

# We put proxy headers in the connection_key, because
# ``proxy_headers`` method might be overridden, so we can't
# rely on proxy headers being the same for the same proxies.
proxy_headers_key = (frozenset(proxy_headers.items())
if proxy_headers else None)
connection_key = (host, port, scheme, cert, verify,
proxy_netloc, proxy_headers_key)
try:
conn = self.connections[(host, port, scheme, cert, verify)]
conn = self.connections[connection_key]
except KeyError:
conn = HTTPConnection(
host,
port,
secure=secure,
ssl_context=ssl_context)
self.connections[(host, port, scheme, cert, verify)] = conn
ssl_context=ssl_context,
proxy_host=proxy_netloc,
proxy_headers=proxy_headers)
self.connections[connection_key] = conn

return conn

def send(self, request, stream=False, cert=None, verify=True, **kwargs):
def send(self, request, stream=False, cert=None, verify=True, proxies=None,
**kwargs):
"""
Sends a HTTP message to the server.
"""
proxy = select_proxy(request.url, proxies)
if proxy:
proxy = prepend_scheme_if_needed(proxy, 'http')

parsed = urlparse(request.url)
conn = self.get_connection(
parsed.hostname,
parsed.port,
parsed.scheme,
cert=cert,
verify=verify)
verify=verify,
proxy=proxy)

# Build the selector.
selector = parsed.path
Expand All @@ -97,7 +122,7 @@ def send(self, request, stream=False, cert=None, verify=True, **kwargs):
def build_response(self, request, resp):
"""
Builds a Requests' response object. This emulates most of the logic of
the standard fuction but deals with the lack of the ``.headers``
the standard function but deals with the lack of the ``.headers``
property on the HTTP20Response object.
Additionally, this function builds in a number of features that are
Expand Down
134 changes: 100 additions & 34 deletions hyper/http11/connection.py
View file
Expand Up @@ -18,9 +18,11 @@
from .response import HTTP11Response
from ..tls import wrap_socket, H2C_PROTOCOL
from ..common.bufsocket import BufferedSocket
from ..common.exceptions import TLSUpgrade, HTTPUpgrade
from ..common.exceptions import TLSUpgrade, HTTPUpgrade, ProxyError
from ..common.headers import HTTPHeaderMap
from ..common.util import to_bytestring, to_host_port_tuple, HTTPVersion
from ..common.util import (
to_bytestring, to_host_port_tuple, to_native_string, HTTPVersion
)
from ..compat import bytes

# We prefer pycohttpparser to the pure-Python interpretation
Expand All @@ -36,6 +38,43 @@
BODY_FLAT = 2


def _create_tunnel(proxy_host, proxy_port, target_host, target_port,
proxy_headers=None):
"""
Sends CONNECT method to a proxy and returns a socket with established
connection to the target.
:returns: socket
"""
conn = HTTP11Connection(proxy_host, proxy_port)
conn.request('CONNECT', '%s:%d' % (target_host, target_port),
headers=proxy_headers)

resp = conn.get_response()
if resp.status != 200:
raise ProxyError(
"Tunnel connection failed: %d %s" %
(resp.status, to_native_string(resp.reason)),
response=resp
)
return conn._sock


def _headers_to_http_header_map(headers):
# TODO turn this to a classmethod of HTTPHeaderMap
headers = headers or {}
if not isinstance(headers, HTTPHeaderMap):
if isinstance(headers, Mapping):
headers = HTTPHeaderMap(headers.items())
elif isinstance(headers, Iterable):
headers = HTTPHeaderMap(headers)
else:
raise ValueError(
'Header argument must be a dictionary or an iterable'
)
return headers


class HTTP11Connection(object):
"""
An object representing a single HTTP/1.1 connection to a server.
Expand All @@ -53,14 +92,16 @@ class HTTP11Connection(object):
:param proxy_host: (optional) The proxy to connect to. This can be an IP
address or a host name and may include a port.
:param proxy_port: (optional) The proxy port to connect to. If not provided
and one also isn't provided in the ``proxy`` parameter,
and one also isn't provided in the ``proxy_host`` parameter,
defaults to 8080.
:param proxy_headers: (optional) The headers to send to a proxy.
"""

version = HTTPVersion.http11

def __init__(self, host, port=None, secure=None, ssl_context=None,
proxy_host=None, proxy_port=None, **kwargs):
proxy_host=None, proxy_port=None, proxy_headers=None,
**kwargs):
if port is None:
self.host, self.port = to_host_port_tuple(host, default_port=80)
else:
Expand All @@ -83,17 +124,21 @@ def __init__(self, host, port=None, secure=None, ssl_context=None,
self.ssl_context = ssl_context
self._sock = None

# Keep the current request method in order to be able to know
# in get_response() what was the request verb.
self._current_request_method = None

# Setup proxy details if applicable.
if proxy_host:
if proxy_port is None:
self.proxy_host, self.proxy_port = to_host_port_tuple(
proxy_host, default_port=8080
)
else:
self.proxy_host, self.proxy_port = proxy_host, proxy_port
if proxy_host and proxy_port is None:
self.proxy_host, self.proxy_port = to_host_port_tuple(
proxy_host, default_port=8080
)
elif proxy_host:
self.proxy_host, self.proxy_port = proxy_host, proxy_port
else:
self.proxy_host = None
self.proxy_port = None
self.proxy_headers = proxy_headers

#: The size of the in-memory buffer used to store data from the
#: network. This is used as a performance optimisation. Increase buffer
Expand All @@ -113,19 +158,28 @@ def connect(self):
:returns: Nothing.
"""
if self._sock is None:
if not self.proxy_host:
host = self.host
port = self.port
else:
host = self.proxy_host
port = self.proxy_port

sock = socket.create_connection((host, port), 5)
if self.proxy_host and self.secure:
# Send http CONNECT method to a proxy and acquire the socket
sock = _create_tunnel(
self.proxy_host,
self.proxy_port,
self.host,
self.port,
proxy_headers=self.proxy_headers
)
elif self.proxy_host:
# Simple http proxy
sock = socket.create_connection(
(self.proxy_host, self.proxy_port),
5
)
else:
sock = socket.create_connection((self.host, self.port), 5)
proto = None

if self.secure:
assert not self.proxy_host, "Proxy with HTTPS not supported."
sock, proto = wrap_socket(sock, host, self.ssl_context)
sock, proto = wrap_socket(sock, self.host, self.ssl_context)

log.debug("Selected protocol: %s", proto)
sock = BufferedSocket(sock, self.network_buffer_size)
Expand Down Expand Up @@ -154,33 +208,37 @@ def request(self, method, url, body=None, headers=None):
:returns: Nothing.
"""

headers = headers or {}

method = to_bytestring(method)
is_connect_method = b'CONNECT' == method.upper()
self._current_request_method = method

if self.proxy_host and not self.secure:
# As per https://tools.ietf.org/html/rfc2068#section-5.1.2:
# The absoluteURI form is required when the request is being made
# to a proxy.
url = self._absolute_http_url(url)
url = to_bytestring(url)

if not isinstance(headers, HTTPHeaderMap):
if isinstance(headers, Mapping):
headers = HTTPHeaderMap(headers.items())
elif isinstance(headers, Iterable):
headers = HTTPHeaderMap(headers)
else:
raise ValueError(
'Header argument must be a dictionary or an iterable'
)
headers = _headers_to_http_header_map(headers)

# Append proxy headers.
if self.proxy_host and not self.secure:
headers.update(
_headers_to_http_header_map(self.proxy_headers).items()
)

if self._sock is None:
self.connect()

if self._send_http_upgrade:
if not is_connect_method and self._send_http_upgrade:
self._add_upgrade_headers(headers)
self._send_http_upgrade = False

# We may need extra headers.
if body:
body_type = self._add_body_headers(headers, body)

if b'host' not in headers:
if not is_connect_method and b'host' not in headers:
headers[b'host'] = self.host

# Begin by emitting the header block.
Expand All @@ -192,13 +250,20 @@ def request(self, method, url, body=None, headers=None):

return

def _absolute_http_url(self, url):
port_part = ':%d' % self.port if self.port != 80 else ''
return 'http://%s%s%s' % (self.host, port_part, url)

def get_response(self):
"""
Returns a response object.
This is an early beta, so the response object is pretty stupid. That's
ok, we'll fix it later.
"""
method = self._current_request_method
self._current_request_method = None

headers = HTTPHeaderMap()

response = None
Expand Down Expand Up @@ -228,7 +293,8 @@ def get_response(self):
response.msg.tobytes(),
headers,
self._sock,
self
self,
method
)

def _send_headers(self, method, url, headers):
Expand Down

0 comments on commit 411c1c9

Please sign in to comment.