From 411c1c90e99a16e9dbf87758cdf080a3c1afb87d Mon Sep 17 00:00:00 2001 From: Kostya Esmukov Date: Sat, 6 May 2017 13:38:16 +0300 Subject: [PATCH] Add HTTPS proxy support Squashed https://github.com/Lukasa/hyper/pull/322 # Conflicts: # hyper/contrib.py --- hyper/common/connection.py | 11 +- hyper/common/exceptions.py | 19 +++ hyper/contrib.py | 41 ++++- hyper/http11/connection.py | 134 +++++++++++---- hyper/http11/response.py | 21 ++- hyper/http20/connection.py | 59 ++++--- hyper/http20/exceptions.py | 3 +- test/server.py | 49 ++++-- test/test_abstraction.py | 6 +- test/test_http11.py | 61 ++++++- test/test_hyper.py | 35 ++++ test/test_integration.py | 278 +++++++++++++++++++++++++++++++- test/test_integration_http11.py | 151 ++++++++++++++++- 13 files changed, 775 insertions(+), 93 deletions(-) diff --git a/hyper/common/connection.py b/hyper/common/connection.py index 507a8ad7..e225852e 100644 --- a/hyper/common/connection.py +++ b/hyper/common/connection.py @@ -44,8 +44,9 @@ class HTTPConnection(object): :param proxy_host: (optional) The proxy to connect to. This can be an IP address or a host name and may include a port. :param proxy_port: (optional) The proxy port to connect to. If not provided - and one also isn't provided in the ``proxy`` parameter, defaults to - 8080. + and one also isn't provided in the ``proxy_host`` parameter, defaults + to 8080. + :param proxy_headers: (optional) The headers to send to a proxy. """ def __init__(self, host, @@ -56,6 +57,7 @@ def __init__(self, ssl_context=None, proxy_host=None, proxy_port=None, + proxy_headers=None, **kwargs): self._host = host @@ -63,12 +65,13 @@ def __init__(self, self._h1_kwargs = { 'secure': secure, 'ssl_context': ssl_context, 'proxy_host': proxy_host, 'proxy_port': proxy_port, - 'enable_push': enable_push + 'proxy_headers': proxy_headers, 'enable_push': enable_push } self._h2_kwargs = { 'window_manager': window_manager, 'enable_push': enable_push, 'secure': secure, 'ssl_context': ssl_context, - 'proxy_host': proxy_host, 'proxy_port': proxy_port + 'proxy_host': proxy_host, 'proxy_port': proxy_port, + 'proxy_headers': proxy_headers } # Add any unexpected kwargs to both dictionaries. diff --git a/hyper/common/exceptions.py b/hyper/common/exceptions.py index 268431ab..78dfefc9 100644 --- a/hyper/common/exceptions.py +++ b/hyper/common/exceptions.py @@ -71,3 +71,22 @@ class MissingCertFile(Exception): The certificate file could not be found. """ pass + + +# Create our own ConnectionError. +try: # pragma: no cover + ConnectionError = ConnectionError +except NameError: # pragma: no cover + class ConnectionError(Exception): + """ + An error occurred during connection to a host. + """ + + +class ProxyError(ConnectionError): + """ + An error occurred during connection to a proxy. + """ + def __init__(self, message, response): + self.response = response + super(ProxyError, self).__init__(message) diff --git a/hyper/contrib.py b/hyper/contrib.py index 5c17352d..9ccf21d3 100644 --- a/hyper/contrib.py +++ b/hyper/contrib.py @@ -9,7 +9,9 @@ from requests.adapters import HTTPAdapter from requests.models import Response from requests.structures import CaseInsensitiveDict - from requests.utils import get_encoding_from_headers + from requests.utils import ( + get_encoding_from_headers, select_proxy, prepend_scheme_if_needed + ) from requests.cookies import extract_cookies_to_jar except ImportError: # pragma: no cover HTTPAdapter = object @@ -29,7 +31,8 @@ def __init__(self, *args, **kwargs): #: A mapping between HTTP netlocs and ``HTTP20Connection`` objects. self.connections = {} - def get_connection(self, host, port, scheme, cert=None, verify=True): + def get_connection(self, host, port, scheme, cert=None, verify=True, + proxy=None): """ Gets an appropriate HTTP/2 connection object based on host/port/scheme/cert tuples. @@ -50,29 +53,51 @@ def get_connection(self, host, port, scheme, cert=None, verify=True): elif verify is not True: ssl_context = init_context(cert_path=verify, cert=cert) + if proxy: + proxy_headers = self.proxy_headers(proxy) + proxy_netloc = urlparse(proxy).netloc + else: + proxy_headers = None + proxy_netloc = None + + # We put proxy headers in the connection_key, because + # ``proxy_headers`` method might be overridden, so we can't + # rely on proxy headers being the same for the same proxies. + proxy_headers_key = (frozenset(proxy_headers.items()) + if proxy_headers else None) + connection_key = (host, port, scheme, cert, verify, + proxy_netloc, proxy_headers_key) try: - conn = self.connections[(host, port, scheme, cert, verify)] + conn = self.connections[connection_key] except KeyError: conn = HTTPConnection( host, port, secure=secure, - ssl_context=ssl_context) - self.connections[(host, port, scheme, cert, verify)] = conn + ssl_context=ssl_context, + proxy_host=proxy_netloc, + proxy_headers=proxy_headers) + self.connections[connection_key] = conn return conn - def send(self, request, stream=False, cert=None, verify=True, **kwargs): + def send(self, request, stream=False, cert=None, verify=True, proxies=None, + **kwargs): """ Sends a HTTP message to the server. """ + proxy = select_proxy(request.url, proxies) + if proxy: + proxy = prepend_scheme_if_needed(proxy, 'http') + parsed = urlparse(request.url) conn = self.get_connection( parsed.hostname, parsed.port, parsed.scheme, cert=cert, - verify=verify) + verify=verify, + proxy=proxy) # Build the selector. selector = parsed.path @@ -97,7 +122,7 @@ def send(self, request, stream=False, cert=None, verify=True, **kwargs): def build_response(self, request, resp): """ Builds a Requests' response object. This emulates most of the logic of - the standard fuction but deals with the lack of the ``.headers`` + the standard function but deals with the lack of the ``.headers`` property on the HTTP20Response object. Additionally, this function builds in a number of features that are diff --git a/hyper/http11/connection.py b/hyper/http11/connection.py index 4de673bb..0603e68f 100644 --- a/hyper/http11/connection.py +++ b/hyper/http11/connection.py @@ -18,9 +18,11 @@ from .response import HTTP11Response from ..tls import wrap_socket, H2C_PROTOCOL from ..common.bufsocket import BufferedSocket -from ..common.exceptions import TLSUpgrade, HTTPUpgrade +from ..common.exceptions import TLSUpgrade, HTTPUpgrade, ProxyError from ..common.headers import HTTPHeaderMap -from ..common.util import to_bytestring, to_host_port_tuple, HTTPVersion +from ..common.util import ( + to_bytestring, to_host_port_tuple, to_native_string, HTTPVersion +) from ..compat import bytes # We prefer pycohttpparser to the pure-Python interpretation @@ -36,6 +38,43 @@ BODY_FLAT = 2 +def _create_tunnel(proxy_host, proxy_port, target_host, target_port, + proxy_headers=None): + """ + Sends CONNECT method to a proxy and returns a socket with established + connection to the target. + + :returns: socket + """ + conn = HTTP11Connection(proxy_host, proxy_port) + conn.request('CONNECT', '%s:%d' % (target_host, target_port), + headers=proxy_headers) + + resp = conn.get_response() + if resp.status != 200: + raise ProxyError( + "Tunnel connection failed: %d %s" % + (resp.status, to_native_string(resp.reason)), + response=resp + ) + return conn._sock + + +def _headers_to_http_header_map(headers): + # TODO turn this to a classmethod of HTTPHeaderMap + headers = headers or {} + if not isinstance(headers, HTTPHeaderMap): + if isinstance(headers, Mapping): + headers = HTTPHeaderMap(headers.items()) + elif isinstance(headers, Iterable): + headers = HTTPHeaderMap(headers) + else: + raise ValueError( + 'Header argument must be a dictionary or an iterable' + ) + return headers + + class HTTP11Connection(object): """ An object representing a single HTTP/1.1 connection to a server. @@ -53,14 +92,16 @@ class HTTP11Connection(object): :param proxy_host: (optional) The proxy to connect to. This can be an IP address or a host name and may include a port. :param proxy_port: (optional) The proxy port to connect to. If not provided - and one also isn't provided in the ``proxy`` parameter, + and one also isn't provided in the ``proxy_host`` parameter, defaults to 8080. + :param proxy_headers: (optional) The headers to send to a proxy. """ version = HTTPVersion.http11 def __init__(self, host, port=None, secure=None, ssl_context=None, - proxy_host=None, proxy_port=None, **kwargs): + proxy_host=None, proxy_port=None, proxy_headers=None, + **kwargs): if port is None: self.host, self.port = to_host_port_tuple(host, default_port=80) else: @@ -83,17 +124,21 @@ def __init__(self, host, port=None, secure=None, ssl_context=None, self.ssl_context = ssl_context self._sock = None + # Keep the current request method in order to be able to know + # in get_response() what was the request verb. + self._current_request_method = None + # Setup proxy details if applicable. - if proxy_host: - if proxy_port is None: - self.proxy_host, self.proxy_port = to_host_port_tuple( - proxy_host, default_port=8080 - ) - else: - self.proxy_host, self.proxy_port = proxy_host, proxy_port + if proxy_host and proxy_port is None: + self.proxy_host, self.proxy_port = to_host_port_tuple( + proxy_host, default_port=8080 + ) + elif proxy_host: + self.proxy_host, self.proxy_port = proxy_host, proxy_port else: self.proxy_host = None self.proxy_port = None + self.proxy_headers = proxy_headers #: The size of the in-memory buffer used to store data from the #: network. This is used as a performance optimisation. Increase buffer @@ -113,19 +158,28 @@ def connect(self): :returns: Nothing. """ if self._sock is None: - if not self.proxy_host: - host = self.host - port = self.port - else: - host = self.proxy_host - port = self.proxy_port - sock = socket.create_connection((host, port), 5) + if self.proxy_host and self.secure: + # Send http CONNECT method to a proxy and acquire the socket + sock = _create_tunnel( + self.proxy_host, + self.proxy_port, + self.host, + self.port, + proxy_headers=self.proxy_headers + ) + elif self.proxy_host: + # Simple http proxy + sock = socket.create_connection( + (self.proxy_host, self.proxy_port), + 5 + ) + else: + sock = socket.create_connection((self.host, self.port), 5) proto = None if self.secure: - assert not self.proxy_host, "Proxy with HTTPS not supported." - sock, proto = wrap_socket(sock, host, self.ssl_context) + sock, proto = wrap_socket(sock, self.host, self.ssl_context) log.debug("Selected protocol: %s", proto) sock = BufferedSocket(sock, self.network_buffer_size) @@ -154,25 +208,29 @@ def request(self, method, url, body=None, headers=None): :returns: Nothing. """ - headers = headers or {} - method = to_bytestring(method) + is_connect_method = b'CONNECT' == method.upper() + self._current_request_method = method + + if self.proxy_host and not self.secure: + # As per https://tools.ietf.org/html/rfc2068#section-5.1.2: + # The absoluteURI form is required when the request is being made + # to a proxy. + url = self._absolute_http_url(url) url = to_bytestring(url) - if not isinstance(headers, HTTPHeaderMap): - if isinstance(headers, Mapping): - headers = HTTPHeaderMap(headers.items()) - elif isinstance(headers, Iterable): - headers = HTTPHeaderMap(headers) - else: - raise ValueError( - 'Header argument must be a dictionary or an iterable' - ) + headers = _headers_to_http_header_map(headers) + + # Append proxy headers. + if self.proxy_host and not self.secure: + headers.update( + _headers_to_http_header_map(self.proxy_headers).items() + ) if self._sock is None: self.connect() - if self._send_http_upgrade: + if not is_connect_method and self._send_http_upgrade: self._add_upgrade_headers(headers) self._send_http_upgrade = False @@ -180,7 +238,7 @@ def request(self, method, url, body=None, headers=None): if body: body_type = self._add_body_headers(headers, body) - if b'host' not in headers: + if not is_connect_method and b'host' not in headers: headers[b'host'] = self.host # Begin by emitting the header block. @@ -192,6 +250,10 @@ def request(self, method, url, body=None, headers=None): return + def _absolute_http_url(self, url): + port_part = ':%d' % self.port if self.port != 80 else '' + return 'http://%s%s%s' % (self.host, port_part, url) + def get_response(self): """ Returns a response object. @@ -199,6 +261,9 @@ def get_response(self): This is an early beta, so the response object is pretty stupid. That's ok, we'll fix it later. """ + method = self._current_request_method + self._current_request_method = None + headers = HTTPHeaderMap() response = None @@ -228,7 +293,8 @@ def get_response(self): response.msg.tobytes(), headers, self._sock, - self + self, + method ) def _send_headers(self, method, url, headers): diff --git a/hyper/http11/response.py b/hyper/http11/response.py index 318ab659..8f3eb985 100644 --- a/hyper/http11/response.py +++ b/hyper/http11/response.py @@ -27,7 +27,8 @@ class HTTP11Response(object): version = HTTPVersion.http11 - def __init__(self, code, reason, headers, sock, connection=None): + def __init__(self, code, reason, headers, sock, connection=None, + request_method=None): #: The reason phrase returned by the server. self.reason = reason @@ -62,11 +63,19 @@ def __init__(self, code, reason, headers, sock, connection=None): b'chunked' in self.headers.get(b'transfer-encoding', []) ) - # One of the following must be true: we must expect that the connection - # will be closed following the body, or that a content-length was sent, - # or that we're getting a chunked response. - # FIXME: Remove naked assert, replace with something better. - assert self._expect_close or self._length is not None or self._chunked + # When content-length is absent and response is not chunked, + # body length is determined by connection closure. + # https://tools.ietf.org/html/rfc7230#section-3.3.3 + if self._length is None and not self._chunked: + # 200 response to a CONNECT request means that proxy has connected + # to the target host and it will start forwarding everything sent + # from the either side. Thus we must not try to read body of this + # response. Socket of current connection will be taken over by + # the code that has sent a CONNECT request. + if not (request_method is not None and + b'CONNECT' == request_method.upper() and + code == 200): + self._expect_close = True # This object is used for decompressing gzipped request bodies. Right # now we only support gzip because that's all the RFC mandates of us. diff --git a/hyper/http20/connection.py b/hyper/http20/connection.py index 8b2a71e8..54a3c99d 100644 --- a/hyper/http20/connection.py +++ b/hyper/http20/connection.py @@ -18,6 +18,7 @@ to_host_port_tuple, to_native_string, to_bytestring, HTTPVersion ) from ..compat import unicode, bytes +from ..http11.connection import _create_tunnel from .stream import Stream from .response import HTTP20Response, HTTP20Push from .window import FlowControlManager @@ -29,6 +30,7 @@ import socket import time import threading +import itertools log = logging.getLogger(__name__) @@ -90,15 +92,17 @@ class HTTP20Connection(object): :param proxy_host: (optional) The proxy to connect to. This can be an IP address or a host name and may include a port. :param proxy_port: (optional) The proxy port to connect to. If not provided - and one also isn't provided in the ``proxy`` parameter, defaults to - 8080. + and one also isn't provided in the ``proxy_host`` parameter, defaults + to 8080. + :param proxy_headers: (optional) The headers to send to a proxy. """ version = HTTPVersion.http20 def __init__(self, host, port=None, secure=None, window_manager=None, enable_push=False, ssl_context=None, proxy_host=None, - proxy_port=None, force_proto=None, **kwargs): + proxy_port=None, force_proto=None, proxy_headers=None, + **kwargs): """ Creates an HTTP/2 connection to a specific server. """ @@ -118,16 +122,16 @@ def __init__(self, host, port=None, secure=None, window_manager=None, self.ssl_context = ssl_context # Setup proxy details if applicable. - if proxy_host: - if proxy_port is None: - self.proxy_host, self.proxy_port = to_host_port_tuple( - proxy_host, default_port=8080 - ) - else: - self.proxy_host, self.proxy_port = proxy_host, proxy_port + if proxy_host and proxy_port is None: + self.proxy_host, self.proxy_port = to_host_port_tuple( + proxy_host, default_port=8080 + ) + elif proxy_host: + self.proxy_host, self.proxy_port = proxy_host, proxy_port else: self.proxy_host = None self.proxy_port = None + self.proxy_headers = proxy_headers #: The size of the in-memory buffer used to store data from the #: network. This is used as a performance optimisation. Increase buffer @@ -272,10 +276,18 @@ def request(self, method, url, body=None, headers=None): # being sent in the wrong order, which can lead to the out-of-order # messages with lower stream IDs being closed prematurely. with self._write_lock: + # Unlike HTTP/1.1, HTTP/2 (according to RFC 7540) doesn't require + # to use absolute URI when proxying. + stream_id = self.putrequest(method, url) default_headers = (':method', ':scheme', ':authority', ':path') - for name, value in headers.items(): + all_headers = headers.items() + if self.proxy_host and not self.secure: + proxy_headers = self.proxy_headers or {} + all_headers = itertools.chain(all_headers, + proxy_headers.items()) + for name, value in all_headers: is_default = to_native_string(name) in default_headers self.putheader(name, value, stream_id, replace=is_default) @@ -358,18 +370,25 @@ def connect(self): if self._sock is not None: return - if not self.proxy_host: - host = self.host - port = self.port + if self.proxy_host and self.secure: + # Send http CONNECT method to a proxy and acquire the socket + sock = _create_tunnel( + self.proxy_host, + self.proxy_port, + self.host, + self.port, + proxy_headers=self.proxy_headers + ) + elif self.proxy_host: + # Simple http proxy + sock = socket.create_connection( + (self.proxy_host, self.proxy_port) + ) else: - host = self.proxy_host - port = self.proxy_port - - sock = socket.create_connection((host, port)) + sock = socket.create_connection((self.host, self.port)) if self.secure: - assert not self.proxy_host, "Proxy with HTTPS not supported." - sock, proto = wrap_socket(sock, host, self.ssl_context, + sock, proto = wrap_socket(sock, self.host, self.ssl_context, force_proto=self.force_proto) else: proto = H2C_PROTOCOL diff --git a/hyper/http20/exceptions.py b/hyper/http20/exceptions.py index 69e25816..634ef284 100644 --- a/hyper/http20/exceptions.py +++ b/hyper/http20/exceptions.py @@ -5,6 +5,7 @@ This defines exceptions used in the HTTP/2 portion of hyper. """ +from ..common.exceptions import ConnectionError as CommonConnectionError class HTTP20Error(Exception): @@ -28,7 +29,7 @@ class HPACKDecodingError(HTTP20Error): pass -class ConnectionError(HTTP20Error): +class ConnectionError(CommonConnectionError, HTTP20Error): """ The remote party signalled an error affecting the entire HTTP/2 connection, and the connection has been closed. diff --git a/test/server.py b/test/server.py index 3f6ded4a..482bf734 100644 --- a/test/server.py +++ b/test/server.py @@ -15,6 +15,7 @@ import threading import socket import sys +from enum import Enum from hyper import HTTP20Connection from hyper.compat import ssl @@ -27,6 +28,23 @@ from hyper.tls import NPN_PROTOCOL +class SocketSecuritySetting(Enum): + """ + Server socket TLS wrapping strategy: + + SECURE - automatically wrap socket + INSECURE - never wrap + SECURE_NO_AUTO_WRAP - init context, but socket must be wrapped manually + + The values are needed to be able to convert ``secure`` boolean flag of + a client to a member of this enum: + ``socket_security = SocketSecuritySetting(secure)`` + """ + SECURE = True + INSECURE = False + SECURE_NO_AUTO_WRAP = 'NO_AUTO_WRAP' + + class SocketServerThread(threading.Thread): """ This method stolen wholesale from shazow/urllib3 under license. See @@ -42,16 +60,17 @@ def __init__(self, host='localhost', ready_event=None, h2=True, - secure=True): + socket_security=SocketSecuritySetting.SECURE): threading.Thread.__init__(self) self.socket_handler = socket_handler self.host = host - self.secure = secure + self.socket_security = socket_security self.ready_event = ready_event self.daemon = True - if self.secure: + if self.socket_security in (SocketSecuritySetting.SECURE, + SocketSecuritySetting.SECURE_NO_AUTO_WRAP): self.cxt = ssl.SSLContext(ssl.PROTOCOL_SSLv23) if ssl.HAS_NPN and h2: self.cxt.set_npn_protocols([NPN_PROTOCOL]) @@ -63,8 +82,8 @@ def _start_server(self): if sys.platform != 'win32': sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - if self.secure: - sock = self.cxt.wrap_socket(sock, server_side=True) + if self.socket_security == SocketSecuritySetting.SECURE: + sock = self.wrap_socket(sock) sock.bind((self.host, 0)) self.port = sock.getsockname()[1] @@ -77,8 +96,8 @@ def _start_server(self): self.socket_handler(sock) sock.close() - def _wrap_socket(self, sock): - raise NotImplementedError() + def wrap_socket(self, sock): + return self.cxt.wrap_socket(sock, server_side=True) def run(self): self.server = self._start_server() @@ -92,7 +111,7 @@ class SocketLevelTest(object): def set_up(self, secure=True, proxy=False): self.host = None self.port = None - self.secure = secure if not proxy else False + self.socket_security = SocketSecuritySetting(secure) self.proxy = proxy self.server_thread = None @@ -105,14 +124,24 @@ def _start_server(self, socket_handler): socket_handler=socket_handler, ready_event=ready_event, h2=self.h2, - secure=self.secure + socket_security=self.socket_security ) self.server_thread.start() ready_event.wait() self.host = self.server_thread.host self.port = self.server_thread.port - self.secure = self.server_thread.secure + self.socket_security = self.server_thread.socket_security + + @property + def secure(self): + return self.socket_security in \ + (SocketSecuritySetting.SECURE, + SocketSecuritySetting.SECURE_NO_AUTO_WRAP) + + @secure.setter + def secure(self, value): + self.socket_security = SocketSecuritySetting(value) def get_connection(self): if self.h2: diff --git a/test/test_abstraction.py b/test/test_abstraction.py index 7c2cad1a..d48b3954 100644 --- a/test/test_abstraction.py +++ b/test/test_abstraction.py @@ -10,7 +10,7 @@ def test_h1_kwargs(self): c = HTTPConnection( 'test', 443, secure=False, window_manager=True, enable_push=True, ssl_context=False, proxy_host=False, proxy_port=False, - other_kwarg=True + proxy_headers=False, other_kwarg=True ) assert c._h1_kwargs == { @@ -18,6 +18,7 @@ def test_h1_kwargs(self): 'ssl_context': False, 'proxy_host': False, 'proxy_port': False, + 'proxy_headers': False, 'other_kwarg': True, 'enable_push': True, } @@ -26,7 +27,7 @@ def test_h2_kwargs(self): c = HTTPConnection( 'test', 443, secure=False, window_manager=True, enable_push=True, ssl_context=True, proxy_host=False, proxy_port=False, - other_kwarg=True + proxy_headers=False, other_kwarg=True ) assert c._h2_kwargs == { @@ -36,6 +37,7 @@ def test_h2_kwargs(self): 'ssl_context': True, 'proxy_host': False, 'proxy_port': False, + 'proxy_headers': False, 'other_kwarg': True, } diff --git a/test/test_http11.py b/test/test_http11.py index 7d4be5b6..27976647 100644 --- a/test/test_http11.py +++ b/test/test_http11.py @@ -170,7 +170,7 @@ def test_proxy_request(self): c.request('GET', '/get', headers={'User-Agent': 'hyper'}) expected = ( - b"GET /get HTTP/1.1\r\n" + b"GET http://httpbin.org/get HTTP/1.1\r\n" b"User-Agent: hyper\r\n" b"connection: Upgrade, HTTP2-Settings\r\n" b"upgrade: h2c\r\n" @@ -182,6 +182,65 @@ def test_proxy_request(self): assert received == expected + def test_proxy_request_with_non_standard_port(self): + c = HTTP11Connection('httpbin.org:8080', proxy_host='localhost') + c._sock = sock = DummySocket() + + c.request('GET', '/get', headers={'User-Agent': 'hyper'}) + + expected = ( + b"GET http://httpbin.org:8080/get HTTP/1.1\r\n" + b"User-Agent: hyper\r\n" + b"connection: Upgrade, HTTP2-Settings\r\n" + b"upgrade: h2c\r\n" + b"HTTP2-Settings: AAQAAP__\r\n" + b"host: httpbin.org\r\n" + b"\r\n" + ) + received = b''.join(sock.queue) + + assert received == expected + + def test_proxy_headers_presence_for_insecure_request(self): + c = HTTP11Connection( + 'httpbin.org', secure=False, proxy_host='localhost', + proxy_headers={'Proxy-Authorization': 'Basic ==='}) + c._sock = sock = DummySocket() + + c.request('GET', '/get', headers={'User-Agent': 'hyper'}) + + expected = ( + b"GET http://httpbin.org/get HTTP/1.1\r\n" + b"User-Agent: hyper\r\n" + b"proxy-authorization: Basic ===\r\n" + b"connection: Upgrade, HTTP2-Settings\r\n" + b"upgrade: h2c\r\n" + b"HTTP2-Settings: AAQAAP__\r\n" + b"host: httpbin.org\r\n" + b"\r\n" + ) + received = b''.join(sock.queue) + + assert received == expected + + def test_proxy_headers_absence_for_secure_request(self): + c = HTTP11Connection( + 'httpbin.org', secure=True, proxy_host='localhost', + proxy_headers={'Proxy-Authorization': 'Basic ==='}) + c._sock = sock = DummySocket() + + c.request('GET', '/get', headers={'User-Agent': 'hyper'}) + + expected = ( + b"GET /get HTTP/1.1\r\n" + b"User-Agent: hyper\r\n" + b"host: httpbin.org\r\n" + b"\r\n" + ) + received = b''.join(sock.queue) + + assert received == expected + def test_request_with_bytestring_body(self): c = HTTP11Connection('httpbin.org') c._sock = sock = DummySocket() diff --git a/test/test_hyper.py b/test/test_hyper.py index fdee058e..0556bb0c 100644 --- a/test/test_hyper.py +++ b/test/test_hyper.py @@ -585,6 +585,41 @@ def test_that_using_proxy_keeps_http_headers_intact(self): (b':path', b'/'), ] + def test_proxy_headers_presence_for_insecure_request(self): + sock = DummySocket() + c = HTTP20Connection( + 'www.google.com', secure=False, proxy_host='localhost', + proxy_headers={'Proxy-Authorization': 'Basic ==='} + ) + c._sock = sock + c.request('GET', '/') + s = c.recent_stream + + assert list(s.headers.items()) == [ + (b':method', b'GET'), + (b':scheme', b'http'), + (b':authority', b'www.google.com'), + (b':path', b'/'), + (b'proxy-authorization', b'Basic ==='), + ] + + def test_proxy_headers_absence_for_secure_request(self): + sock = DummySocket() + c = HTTP20Connection( + 'www.google.com', secure=True, proxy_host='localhost', + proxy_headers={'Proxy-Authorization': 'Basic ==='} + ) + c._sock = sock + c.request('GET', '/') + s = c.recent_stream + + assert list(s.headers.items()) == [ + (b':method', b'GET'), + (b':scheme', b'https'), + (b':authority', b'www.google.com'), + (b':path', b'/'), + ] + def test_recv_cb_n_times(self): sock = DummySocket() sock.can_read = True diff --git a/test/test_integration.py b/test/test_integration.py index 72da6156..05931274 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -6,6 +6,7 @@ This file defines integration-type tests for hyper. These are still not fully hitting the network, so that's alright. """ +import base64 import requests import threading import time @@ -17,7 +18,8 @@ from h2.frame_buffer import FrameBuffer from hyper.compat import ssl from hyper.contrib import HTTP20Adapter -from hyper.common.util import HTTPVersion +from hyper.common.exceptions import ProxyError +from hyper.common.util import HTTPVersion, to_bytestring from hyperframe.frame import ( Frame, SettingsFrame, WindowUpdateFrame, DataFrame, HeadersFrame, GoAwayFrame, RstStreamFrame @@ -28,7 +30,7 @@ REQUEST_CODES, REQUEST_CODES_LENGTH ) from hyper.http20.exceptions import ConnectionError, StreamResetError -from server import SocketLevelTest +from server import SocketLevelTest, SocketSecuritySetting # Turn off certificate verification for the tests. if ssl is not None: @@ -620,8 +622,8 @@ def socket_handler(listener): recv_event.set() self.tear_down() - def test_proxy_connection(self): - self.set_up(proxy=True) + def test_insecure_proxy_connection(self): + self.set_up(secure=False, proxy=True) data = [] req_event = threading.Event() @@ -672,6 +674,107 @@ def socket_handler(listener): recv_event.set() self.tear_down() + def test_secure_proxy_connection(self): + self.set_up(secure=SocketSecuritySetting.SECURE_NO_AUTO_WRAP, + proxy=True) + + data = [] + connect_request_headers = [] + req_event = threading.Event() + recv_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # Read the CONNECT request + while not b''.join(connect_request_headers).endswith(b'\r\n\r\n'): + connect_request_headers.append(sock.recv(65535)) + + sock.send(b'HTTP/1.0 200 Connection established\r\n\r\n') + + sock = self.server_thread.wrap_socket(sock) + + receive_preamble(sock) + + data.append(sock.recv(65535)) + req_event.wait(5) + + h = HeadersFrame(1) + h.data = self.get_encoder().encode( + [ + (':status', 200), + ('content-type', 'not/real'), + ('content-length', 12), + ('server', 'socket-level-server') + ] + ) + h.flags.add('END_HEADERS') + sock.send(h.serialize()) + + d = DataFrame(1) + d.data = b'thisisaproxy' + d.flags.add('END_STREAM') + sock.send(d.serialize()) + + recv_event.wait(5) + sock.close() + + self._start_server(socket_handler) + c = self.get_connection() + c.request('GET', '/') + req_event.set() + r = c.get_response() + + assert r.status == 200 + assert len(r.headers) == 3 + assert r.headers[b'server'] == [b'socket-level-server'] + assert r.headers[b'content-length'] == [b'12'] + assert r.headers[b'content-type'] == [b'not/real'] + + assert r.read() == b'thisisaproxy' + + assert (to_bytestring( + 'CONNECT %s:%d HTTP/1.1\r\n\r\n' % (c.host, c.port)) == + b''.join(connect_request_headers)) + + recv_event.set() + self.tear_down() + + def test_failing_proxy_tunnel(self): + self.set_up(secure=SocketSecuritySetting.SECURE_NO_AUTO_WRAP, + proxy=True) + + recv_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # Read the CONNECT request + connect_data = b'' + while not connect_data.endswith(b'\r\n\r\n'): + connect_data += sock.recv(65535) + + sock.send(b'HTTP/1.0 407 Proxy Authentication Required\r\n\r\n') + + recv_event.wait(5) + sock.close() + + self._start_server(socket_handler) + conn = self.get_connection() + + try: + conn.connect() + assert False, "Exception should have been thrown" + except ProxyError as e: + assert e.response.status == 407 + assert e.response.reason == b'Proxy Authentication Required' + + # Confirm the connection is closed. + assert conn._sock is None + + recv_event.set() + self.tear_down() + def test_resetting_stream_with_frames_in_flight(self): """ Hyper emits only one RST_STREAM frame, despite the other frames in @@ -1229,3 +1332,170 @@ def socket_handler(listener): recv_event.set() self.tear_down() + + def test_adapter_uses_proxies(self): + self.set_up(secure=SocketSecuritySetting.SECURE_NO_AUTO_WRAP, + proxy=True) + + send_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # Read the CONNECT request + connect_data = b'' + while not connect_data.endswith(b'\r\n\r\n'): + connect_data += sock.recv(65535) + + sock.send(b'HTTP/1.0 200 Connection established\r\n\r\n') + + sock = self.server_thread.wrap_socket(sock) + + # We should get the initial request. + data = b'' + while not data.endswith(b'\r\n\r\n'): + data += sock.recv(65535) + + send_event.wait() + + # We need to send back a response. + resp = ( + b'HTTP/1.1 201 No Content\r\n' + b'Server: socket-level-server\r\n' + b'Content-Length: 0\r\n' + b'Connection: close\r\n' + b'\r\n' + ) + sock.send(resp) + + sock.close() + + self._start_server(socket_handler) + s = requests.Session() + s.proxies = {'all': 'http://%s:%s' % (self.host, self.port)} + s.mount('https://', HTTP20Adapter()) + send_event.set() + r = s.get('https://foobar/') + + assert r.status_code == 201 + assert len(r.headers) == 3 + assert r.headers[b'server'] == b'socket-level-server' + assert r.headers[b'content-length'] == b'0' + assert r.headers[b'connection'] == b'close' + + assert r.content == b'' + + self.tear_down() + + def test_adapter_uses_proxy_auth_for_secure(self): + self.set_up(secure=SocketSecuritySetting.SECURE_NO_AUTO_WRAP, + proxy=True) + + send_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # Read the CONNECT request + connect_data = b'' + while not connect_data.endswith(b'\r\n\r\n'): + connect_data += sock.recv(65535) + + # Ensure that request contains the proper Proxy-Authorization + # header + assert (b'CONNECT foobar:443 HTTP/1.1\r\n' + b'Proxy-Authorization: Basic ' + + base64.b64encode(b'foo:bar') + b'\r\n' + b'\r\n') == connect_data + + sock.send(b'HTTP/1.0 200 Connection established\r\n\r\n') + + sock = self.server_thread.wrap_socket(sock) + + # We should get the initial request. + data = b'' + while not data.endswith(b'\r\n\r\n'): + data += sock.recv(65535) + # Ensure that proxy headers are not passed via tunnelled connection + assert b'Proxy-Authorization:' not in data + + send_event.wait() + + # We need to send back a response. + resp = ( + b'HTTP/1.1 201 No Content\r\n' + b'Server: socket-level-server\r\n' + b'Content-Length: 0\r\n' + b'Connection: close\r\n' + b'\r\n' + ) + sock.send(resp) + + sock.close() + + self._start_server(socket_handler) + s = requests.Session() + s.proxies = {'all': 'http://foo:bar@%s:%s' % (self.host, self.port)} + s.mount('https://', HTTP20Adapter()) + send_event.set() + r = s.get('https://foobar/') + + assert r.status_code == 201 + assert len(r.headers) == 3 + assert r.headers[b'server'] == b'socket-level-server' + assert r.headers[b'content-length'] == b'0' + assert r.headers[b'connection'] == b'close' + + assert r.content == b'' + + self.tear_down() + + def test_adapter_uses_proxy_auth_for_insecure(self): + self.set_up(secure=False, proxy=True) + + send_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # We should get the initial request. + connect_data = b'' + while not connect_data.endswith(b'\r\n\r\n'): + connect_data += sock.recv(65535) + + # Ensure that request contains the proper Proxy-Authorization + # header + assert (b'Proxy-Authorization: Basic ' + + base64.b64encode(b'foo:bar') + b'\r\n' + ).lower() in connect_data.lower() + + send_event.wait() + + # We need to send back a response. + resp = ( + b'HTTP/1.1 201 No Content\r\n' + b'Server: socket-level-server\r\n' + b'Content-Length: 0\r\n' + b'Connection: close\r\n' + b'\r\n' + ) + sock.send(resp) + + sock.close() + + self._start_server(socket_handler) + s = requests.Session() + s.proxies = {'all': 'http://foo:bar@%s:%s' % (self.host, self.port)} + s.mount('http://', HTTP20Adapter()) + send_event.set() + r = s.get('http://foobar/') + + assert r.status_code == 201 + assert len(r.headers) == 3 + assert r.headers[b'server'] == b'socket-level-server' + assert r.headers[b'content-length'] == b'0' + assert r.headers[b'connection'] == b'close' + + assert r.content == b'' + + self.tear_down() diff --git a/test/test_integration_http11.py b/test/test_integration_http11.py index 8c70f266..ee318797 100644 --- a/test/test_integration_http11.py +++ b/test/test_integration_http11.py @@ -11,8 +11,9 @@ import pytest from hyper.compat import ssl -from server import SocketLevelTest +from server import SocketLevelTest, SocketSecuritySetting from hyper.common.exceptions import HTTPUpgrade +from hyper.common.util import to_bytestring # Turn off certificate verification for the tests. if ssl is not None: @@ -119,14 +120,110 @@ def socket_handler(listener): assert r.read() == b'hellotheresirfinalfantasy' - def test_proxy_request_response(self): - self.set_up(proxy=True) + def test_closing_response_without_headers(self): + self.set_up() + + send_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # We should get the initial request. + data = b'' + while not data.endswith(b'\r\n\r\n'): + data += sock.recv(65535) + + send_event.wait() + + # We need to send back a response. + resp = ( + b'HTTP/1.1 200 OK\r\n' + b'Server: socket-level-server\r\n' + b'\r\n' + ) + sock.send(resp) + + sock.send(b'hi') + + sock.close() + + self._start_server(socket_handler) + c = self.get_connection() + c.request('GET', '/') + send_event.set() + r = c.get_response() + + assert r.status == 200 + assert r.reason == b'OK' + assert len(r.headers) == 1 + assert r.headers[b'server'] == [b'socket-level-server'] + + assert r.read() == b'hi' + + assert c._sock is None + + def test_insecure_proxy_request_response(self): + self.set_up(secure=False, proxy=True) + + send_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # We should get the initial request. + data = b'' + while not data.endswith(b'\r\n\r\n'): + data += sock.recv(65535) + + send_event.wait() + + # We need to send back a response. + resp = ( + b'HTTP/1.1 201 No Content\r\n' + b'Server: socket-level-server\r\n' + b'Content-Length: 0\r\n' + b'Connection: close\r\n' + b'\r\n' + ) + sock.send(resp) + + sock.close() + self._start_server(socket_handler) + c = self.get_connection() + c.request('GET', '/') + send_event.set() + r = c.get_response() + + assert r.status == 201 + assert r.reason == b'No Content' + assert len(r.headers) == 3 + assert r.headers[b'server'] == [b'socket-level-server'] + assert r.headers[b'content-length'] == [b'0'] + assert r.headers[b'connection'] == [b'close'] + + assert r.read() == b'' + + assert c._sock is None + + def test_secure_proxy_request_response(self): + self.set_up(secure=SocketSecuritySetting.SECURE_NO_AUTO_WRAP, + proxy=True) + + connect_request_headers = [] send_event = threading.Event() def socket_handler(listener): sock = listener.accept()[0] + # Read the CONNECT request + while not b''.join(connect_request_headers).endswith(b'\r\n\r\n'): + connect_request_headers.append(sock.recv(65535)) + + sock.send(b'HTTP/1.0 200 Connection established\r\n\r\n') + + sock = self.server_thread.wrap_socket(sock) + # We should get the initial request. data = b'' while not data.endswith(b'\r\n\r\n'): @@ -161,8 +258,56 @@ def socket_handler(listener): assert r.read() == b'' + assert (to_bytestring( + 'CONNECT %s:%d HTTP/1.1\r\n\r\n' % (c.host, c.port)) == + b''.join(connect_request_headers)) + assert c._sock is None + def test_proxy_connection_close_is_respected(self): + self.set_up(secure=False, proxy=True) + + send_event = threading.Event() + + def socket_handler(listener): + sock = listener.accept()[0] + + # We should get the initial request. + data = b'' + while not data.endswith(b'\r\n\r\n'): + data += sock.recv(65535) + + send_event.wait() + + # We need to send back a response. + resp = ( + b'HTTP/1.0 407 Proxy Authentication Required\r\n' + b'Proxy-Authenticate: Basic realm="proxy"\r\n' + b'Proxy-Connection: close\r\n' + b'\r\n' + ) + sock.send(resp) + + sock.close() + + self._start_server(socket_handler) + conn = self.get_connection() + conn.request('GET', '/') + send_event.set() + + r = conn.get_response() + + assert r.status == 407 + assert r.reason == b'Proxy Authentication Required' + assert len(r.headers) == 2 + assert r.headers[b'proxy-authenticate'] == [b'Basic realm="proxy"'] + assert r.headers[b'proxy-connection'] == [b'close'] + + assert r.read() == b'' + + # Confirm the connection is closed. + assert conn._sock is None + def test_response_with_body(self): self.set_up()