Fix for urllib with python 3.7+ & 3.8+

proxy/common/constants.py

@@ -8,10 +8,9 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+import ipaddress
 import os
 import time
-import ipaddress
-
 from typing import List
 
 from .version import __version__
@@ -73,3 +72,4 @@
 DEFAULT_TIMEOUT = 10
 DEFAULT_VERSION = False
 DEFAULT_HTTP_PORT = 80
+DEFAULT_HTTPS_PORT = 443

proxy/http/parser.py

@@ -8,15 +8,15 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
+from typing import Dict, List, NamedTuple, Optional, Tuple, Type, TypeVar
 from urllib import parse as urlparse
-from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List
 
-from .methods import httpMethods
-from .chunk_parser import ChunkParser, chunkParserStates
-
-from ..common.constants import DEFAULT_DISABLE_HEADERS, COLON, CRLF, WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT
+from ..common.constants import (COLON, CRLF, DEFAULT_DISABLE_HEADERS,
+                                DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT,
+                                HTTP_1_1, WHITESPACE)
 from ..common.utils import build_http_request, find_http_line, text_
-
+from .chunk_parser import ChunkParser, chunkParserStates
+from .methods import httpMethods
 
 HttpParserStates = NamedTuple('HttpParserStates', [
     ('INITIALIZED', int),
@@ -111,8 +111,8 @@ def set_url(self, url: bytes) -> None:
     def set_line_attributes(self) -> None:
         if self.type == httpParserTypes.REQUEST_PARSER:
             if self.method == httpMethods.CONNECT and self.url:
-                u = urlparse.urlsplit(b'//' + self.url.path)
-                self.host, self.port = u.hostname, u.port
+                self.host, self.port = self.url.hostname, self.url.port \
+                    if self.url.port else DEFAULT_HTTPS_PORT
             elif self.url:
                 self.host, self.port = self.url.hostname, self.url.port \
                     if self.url.port else DEFAULT_HTTP_PORT
@@ -164,7 +164,8 @@ def parse(self, raw: bytes) -> None:
                         self.state = httpParserStates.COMPLETE
                     more = False
                 else:
-                    raise NotImplementedError('Parser shouldn\'t have reached here')
+                    raise NotImplementedError(
+                        'Parser shouldn\'t have reached here')
             else:
                 more, raw = self.process(raw)
         self.buffer = raw

tests/http/test_http_parser.py

@@ -11,10 +11,11 @@
 import unittest
 
 from proxy.common.constants import CRLF
-from proxy.common.utils import build_http_request, find_http_line, build_http_response, build_http_header, bytes_
-from proxy.http.methods import httpMethods
+from proxy.common.utils import (build_http_header, build_http_request,
+                                build_http_response, bytes_, find_http_line)
 from proxy.http.codes import httpStatusCodes
-from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates
+from proxy.http.methods import httpMethods
+from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes
 
 
 class TestHttpParser(unittest.TestCase):
@@ -134,7 +135,8 @@ def test_get_full_parse(self) -> None:
         self.assertEqual(self.parser.url.port, None)
         self.assertEqual(self.parser.version, b'HTTP/1.1')
         self.assertEqual(self.parser.state, httpParserStates.COMPLETE)
-        self.assertEqual(self.parser.headers[b'host'], (b'Host', b'example.com'))
+        self.assertEqual(
+            self.parser.headers[b'host'], (b'Host', b'example.com'))
         self.parser.del_headers([b'host'])
         self.parser.add_headers([(b'Host', b'example.com')])
         self.assertEqual(
@@ -188,7 +190,8 @@ def test_get_partial_parse1(self) -> None:
         self.parser.parse(CRLF * 2)
         self.assertEqual(self.parser.total_size, len(pkt) +
                          (3 * len(CRLF)) + len(host_hdr))
-        self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
+        self.assertEqual(
+            self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
         self.assertEqual(self.parser.state, httpParserStates.COMPLETE)
 
     def test_get_partial_parse2(self) -> None:
@@ -205,7 +208,8 @@ def test_get_partial_parse2(self) -> None:
         self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD)
 
         self.parser.parse(b'localhost:8080' + CRLF)
-        self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
+        self.assertEqual(
+            self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
         self.assertEqual(self.parser.buffer, b'')
         self.assertEqual(
             self.parser.state,
@@ -524,3 +528,31 @@ def test_paramiko_doc(self) -> None:
         self.parser = HttpParser(httpParserTypes.RESPONSE_PARSER)
         self.parser.parse(response)
         self.assertEqual(self.parser.state, httpParserStates.COMPLETE)
+
+    def test_set_url_http(self) -> None:
+        self.parser.set_url(b"http://unicorn.fr:8899/path1/path2")
+        self.assertEqual(self.parser.port, 8899)
+        self.assertEqual(self.parser.host, b"unicorn.fr")
+        self.assertEqual(self.parser.path, b"/path1/path2")
+
+        self.parser.set_url(b"http://unicorn.fr/path1/path2")
+        self.assertEqual(self.parser.port, 80)
+        self.assertEqual(self.parser.host, b"unicorn.fr")
+        self.assertEqual(self.parser.path, b"/path1/path2")
+
+    def test_set_url_https(self) -> None:
+        '''
+         The code is setting method as httpMethods.CONNECT when there is HTTPS,
+         here we set it to just test the set_url methdod
+        '''
+        self.parser.method = httpMethods.CONNECT
+
+        self.parser.set_url(b"https://unicorn.fr:443/path1/path2")
+        self.assertEqual(self.parser.port, 443)
+        self.assertEqual(self.parser.host, b"unicorn.fr")
+        self.assertEqual(self.parser.path, b"/path1/path2")
+
+        self.parser.set_url(b"https://unicorn.fr/path1/path2")
+        self.assertEqual(self.parser.port, 443)
+        self.assertEqual(self.parser.host, b"unicorn.fr")
+        self.assertEqual(self.parser.path, b"/path1/path2")

tests/http/test_http_proxy_tls_interception.py

@@ -8,21 +8,20 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-import uuid
-import unittest
+import selectors
 import socket
 import ssl
-import selectors
-
+import unittest
+import uuid
 from typing import Any
 from unittest import mock
 
+from proxy.common.flags import Flags
+from proxy.common.utils import build_http_request, bytes_
 from proxy.core.connection import TcpClientConnection
 from proxy.http.handler import HttpProtocolHandler
-from proxy.http.proxy import HttpProxyPlugin
 from proxy.http.methods import httpMethods
-from proxy.common.utils import build_http_request, bytes_
-from proxy.common.flags import Flags
+from proxy.http.proxy import HttpProxyPlugin
 
 
 class TestHttpProxyTlsInterception(unittest.TestCase):
@@ -42,7 +41,7 @@ def test_e2e(
             mock_ssl_context: mock.Mock,
             mock_ssl_wrap: mock.Mock) -> None:
         host, port = uuid.uuid4().hex, 443
-        netloc = '{0}:{1}'.format(host, port)
+        netloc = 'https://{0}:{1}'.format(host, port)
 
         self.mock_fromfd = mock_fromfd
         self.mock_selector = mock_selector

tests/http/test_protocol_handler.py

@@ -8,23 +8,22 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-import unittest
-import selectors
 import base64
-
+import selectors
+import unittest
 from typing import cast
 from unittest import mock
 
-from proxy.common.version import __version__
+from proxy.common.constants import CRLF
 from proxy.common.flags import Flags
 from proxy.common.utils import bytes_
-from proxy.common.constants import CRLF
+from proxy.common.version import __version__
 from proxy.core.connection import TcpClientConnection
-from proxy.http.parser import HttpParser
-from proxy.http.proxy import HttpProxyPlugin
-from proxy.http.parser import httpParserStates, httpParserTypes
-from proxy.http.exception import ProxyAuthenticationFailed, ProxyConnectionFailed
+from proxy.http.exception import (ProxyAuthenticationFailed,
+                                  ProxyConnectionFailed)
 from proxy.http.handler import HttpProtocolHandler
+from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes
+from proxy.http.proxy import HttpProxyPlugin
 
 
 class TestHttpProtocolHandler(unittest.TestCase):
@@ -134,7 +133,7 @@ def has_buffer() -> bool:
 
         assert self.http_server_port is not None
         self._conn.recv.return_value = CRLF.join([
-            b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port,
+            b'CONNECT https://localhost:%d HTTP/1.1' % self.http_server_port,
             b'Host: localhost:%d' % self.http_server_port,
             b'User-Agent: proxy.py/%s' % bytes_(__version__),
             b'Proxy-Connection: Keep-Alive',
@@ -262,7 +261,7 @@ def test_authenticated_proxy_http_tunnel(
 
         assert self.http_server_port is not None
         self._conn.recv.return_value = CRLF.join([
-            b'CONNECT localhost:%d HTTP/1.1' % self.http_server_port,
+            b'CONNECT https://localhost:%d HTTP/1.1' % self.http_server_port,
             b'Host: localhost:%d' % self.http_server_port,
             b'User-Agent: proxy.py/%s' % bytes_(__version__),
             b'Proxy-Connection: Keep-Alive',

tests/plugin/test_http_proxy_plugins_with_tls_interception.py

@@ -8,21 +8,19 @@
     :copyright: (c) 2013-present by Abhinav Singh and contributors.
     :license: BSD, see LICENSE for more details.
 """
-import unittest
-import socket
 import selectors
+import socket
 import ssl
-
-from unittest import mock
+import unittest
 from typing import Any, cast
+from unittest import mock
 
-from proxy.common.utils import bytes_
 from proxy.common.flags import Flags
-from proxy.common.utils import build_http_request, build_http_response
+from proxy.common.utils import build_http_request, build_http_response, bytes_
 from proxy.core.connection import TcpClientConnection
 from proxy.http.codes import httpStatusCodes
-from proxy.http.methods import httpMethods
 from proxy.http.handler import HttpProtocolHandler
+from proxy.http.methods import httpMethods
 from proxy.http.proxy import HttpProxyPlugin
 
 from .utils import get_plugin_by_test_name
@@ -122,7 +120,7 @@ def send(raw: bytes) -> int:
 
         self._conn.send.side_effect = send
         self._conn.recv.return_value = build_http_request(
-            httpMethods.CONNECT, b'uni.corn:443'
+            httpMethods.CONNECT, b'https://uni.corn:443'
         )
         self.protocol_handler.run_once()