From 84644e99d5a92fe631599c5329f022b06a4f5e03 Mon Sep 17 00:00:00 2001 From: Benoit Date: Tue, 21 Jan 2020 11:35:36 +0100 Subject: [PATCH] Fix for urllib with python 3.7+ & 3.8+ --- proxy/common/constants.py | 4 +- proxy/http/parser.py | 19 ++++---- tests/http/test_http_parser.py | 44 ++++++++++++++++--- .../http/test_http_proxy_tls_interception.py | 15 +++---- tests/http/test_protocol_handler.py | 21 +++++---- ...ttp_proxy_plugins_with_tls_interception.py | 14 +++--- 6 files changed, 73 insertions(+), 44 deletions(-) diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 72efbce57d..46a6c5fb24 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -8,10 +8,9 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import ipaddress import os import time -import ipaddress - from typing import List from .version import __version__ @@ -73,3 +72,4 @@ DEFAULT_TIMEOUT = 10 DEFAULT_VERSION = False DEFAULT_HTTP_PORT = 80 +DEFAULT_HTTPS_PORT = 443 diff --git a/proxy/http/parser.py b/proxy/http/parser.py index ece1ce0861..0e2ef2d3dc 100644 --- a/proxy/http/parser.py +++ b/proxy/http/parser.py @@ -8,15 +8,15 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +from typing import Dict, List, NamedTuple, Optional, Tuple, Type, TypeVar from urllib import parse as urlparse -from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List -from .methods import httpMethods -from .chunk_parser import ChunkParser, chunkParserStates - -from ..common.constants import DEFAULT_DISABLE_HEADERS, COLON, CRLF, WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT +from ..common.constants import (COLON, CRLF, DEFAULT_DISABLE_HEADERS, + DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, + HTTP_1_1, WHITESPACE) from ..common.utils import build_http_request, find_http_line, text_ - +from .chunk_parser import ChunkParser, chunkParserStates +from .methods import httpMethods HttpParserStates = NamedTuple('HttpParserStates', [ ('INITIALIZED', int), @@ -111,8 +111,8 @@ def set_url(self, url: bytes) -> None: def set_line_attributes(self) -> None: if self.type == httpParserTypes.REQUEST_PARSER: if self.method == httpMethods.CONNECT and self.url: - u = urlparse.urlsplit(b'//' + self.url.path) - self.host, self.port = u.hostname, u.port + self.host, self.port = self.url.hostname, self.url.port \ + if self.url.port else DEFAULT_HTTPS_PORT elif self.url: self.host, self.port = self.url.hostname, self.url.port \ if self.url.port else DEFAULT_HTTP_PORT @@ -164,7 +164,8 @@ def parse(self, raw: bytes) -> None: self.state = httpParserStates.COMPLETE more = False else: - raise NotImplementedError('Parser shouldn\'t have reached here') + raise NotImplementedError( + 'Parser shouldn\'t have reached here') else: more, raw = self.process(raw) self.buffer = raw diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py index 6867749c3b..fd0f9e78cb 100644 --- a/tests/http/test_http_parser.py +++ b/tests/http/test_http_parser.py @@ -11,10 +11,11 @@ import unittest from proxy.common.constants import CRLF -from proxy.common.utils import build_http_request, find_http_line, build_http_response, build_http_header, bytes_ -from proxy.http.methods import httpMethods +from proxy.common.utils import (build_http_header, build_http_request, + build_http_response, bytes_, find_http_line) from proxy.http.codes import httpStatusCodes -from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates +from proxy.http.methods import httpMethods +from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes class TestHttpParser(unittest.TestCase): @@ -134,7 +135,8 @@ def test_get_full_parse(self) -> None: self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) - self.assertEqual(self.parser.headers[b'host'], (b'Host', b'example.com')) + self.assertEqual( + self.parser.headers[b'host'], (b'Host', b'example.com')) self.parser.del_headers([b'host']) self.parser.add_headers([(b'Host', b'example.com')]) self.assertEqual( @@ -188,7 +190,8 @@ def test_get_partial_parse1(self) -> None: self.parser.parse(CRLF * 2) self.assertEqual(self.parser.total_size, len(pkt) + (3 * len(CRLF)) + len(host_hdr)) - self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080')) + self.assertEqual( + self.parser.headers[b'host'], (b'Host', b'localhost:8080')) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_get_partial_parse2(self) -> None: @@ -205,7 +208,8 @@ def test_get_partial_parse2(self) -> None: self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) self.parser.parse(b'localhost:8080' + CRLF) - self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080')) + self.assertEqual( + self.parser.headers[b'host'], (b'Host', b'localhost:8080')) self.assertEqual(self.parser.buffer, b'') self.assertEqual( self.parser.state, @@ -524,3 +528,31 @@ def test_paramiko_doc(self) -> None: self.parser = HttpParser(httpParserTypes.RESPONSE_PARSER) self.parser.parse(response) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) + + def test_set_url_http(self) -> None: + self.parser.set_url(b"http://unicorn.fr:8899/path1/path2") + self.assertEqual(self.parser.port, 8899) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2") + + self.parser.set_url(b"http://unicorn.fr/path1/path2") + self.assertEqual(self.parser.port, 80) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2") + + def test_set_url_https(self) -> None: + ''' + The code is setting method as httpMethods.CONNECT when there is HTTPS, + here we set it to just test the set_url methdod + ''' + self.parser.method = httpMethods.CONNECT + + self.parser.set_url(b"https://unicorn.fr:443/path1/path2") + self.assertEqual(self.parser.port, 443) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2") + + self.parser.set_url(b"https://unicorn.fr/path1/path2") + self.assertEqual(self.parser.port, 443) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2") diff --git a/tests/http/test_http_proxy_tls_interception.py b/tests/http/test_http_proxy_tls_interception.py index ff01a7c85f..b6f9888ee5 100644 --- a/tests/http/test_http_proxy_tls_interception.py +++ b/tests/http/test_http_proxy_tls_interception.py @@ -8,21 +8,20 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import uuid -import unittest +import selectors import socket import ssl -import selectors - +import unittest +import uuid from typing import Any from unittest import mock +from proxy.common.flags import Flags +from proxy.common.utils import build_http_request, bytes_ from proxy.core.connection import TcpClientConnection from proxy.http.handler import HttpProtocolHandler -from proxy.http.proxy import HttpProxyPlugin from proxy.http.methods import httpMethods -from proxy.common.utils import build_http_request, bytes_ -from proxy.common.flags import Flags +from proxy.http.proxy import HttpProxyPlugin class TestHttpProxyTlsInterception(unittest.TestCase): @@ -42,7 +41,7 @@ def test_e2e( mock_ssl_context: mock.Mock, mock_ssl_wrap: mock.Mock) -> None: host, port = uuid.uuid4().hex, 443 - netloc = '{0}:{1}'.format(host, port) + netloc = 'https://{0}:{1}'.format(host, port) self.mock_fromfd = mock_fromfd self.mock_selector = mock_selector diff --git a/tests/http/test_protocol_handler.py b/tests/http/test_protocol_handler.py index ca4dac037d..01cf316146 100644 --- a/tests/http/test_protocol_handler.py +++ b/tests/http/test_protocol_handler.py @@ -8,23 +8,22 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import unittest -import selectors import base64 - +import selectors +import unittest from typing import cast from unittest import mock -from proxy.common.version import __version__ +from proxy.common.constants import CRLF from proxy.common.flags import Flags from proxy.common.utils import bytes_ -from proxy.common.constants import CRLF +from proxy.common.version import __version__ from proxy.core.connection import TcpClientConnection -from proxy.http.parser import HttpParser -from proxy.http.proxy import HttpProxyPlugin -from proxy.http.parser import httpParserStates, httpParserTypes -from proxy.http.exception import ProxyAuthenticationFailed, ProxyConnectionFailed +from proxy.http.exception import (ProxyAuthenticationFailed, + ProxyConnectionFailed) from proxy.http.handler import HttpProtocolHandler +from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes +from proxy.http.proxy import HttpProxyPlugin class TestHttpProtocolHandler(unittest.TestCase): @@ -134,7 +133,7 @@ def has_buffer() -> bool: assert self.http_server_port is not None self._conn.recv.return_value = CRLF.join([ - b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port, + b'CONNECT https://localhost:%d HTTP/1.1' % self.http_server_port, b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % bytes_(__version__), b'Proxy-Connection: Keep-Alive', @@ -262,7 +261,7 @@ def test_authenticated_proxy_http_tunnel( assert self.http_server_port is not None self._conn.recv.return_value = CRLF.join([ - b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port, + b'CONNECT https://localhost:%d HTTP/1.1' % self.http_server_port, b'Host: localhost:%d' % self.http_server_port, b'User-Agent: proxy.py/%s' % bytes_(__version__), b'Proxy-Connection: Keep-Alive', diff --git a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py index 2976869d97..e57418d68c 100644 --- a/tests/plugin/test_http_proxy_plugins_with_tls_interception.py +++ b/tests/plugin/test_http_proxy_plugins_with_tls_interception.py @@ -8,21 +8,19 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ -import unittest -import socket import selectors +import socket import ssl - -from unittest import mock +import unittest from typing import Any, cast +from unittest import mock -from proxy.common.utils import bytes_ from proxy.common.flags import Flags -from proxy.common.utils import build_http_request, build_http_response +from proxy.common.utils import build_http_request, build_http_response, bytes_ from proxy.core.connection import TcpClientConnection from proxy.http.codes import httpStatusCodes -from proxy.http.methods import httpMethods from proxy.http.handler import HttpProtocolHandler +from proxy.http.methods import httpMethods from proxy.http.proxy import HttpProxyPlugin from .utils import get_plugin_by_test_name @@ -122,7 +120,7 @@ def send(raw: bytes) -> int: self._conn.send.side_effect = send self._conn.recv.return_value = build_http_request( - httpMethods.CONNECT, b'uni.corn:443' + httpMethods.CONNECT, b'https://uni.corn:443' ) self.protocol_handler.run_once()