From 99b9c8b941a06f9a532b451ab9fe3b76b0df490d Mon Sep 17 00:00:00 2001 From: Benoit Date: Tue, 21 Jan 2020 11:35:36 +0100 Subject: [PATCH] Fix for urllib with python 3.7+ & 3.8+ --- proxy/common/constants.py | 4 +-- proxy/http/parser.py | 22 ++++++++-------- requirements-testing.txt | 1 - tests/http/test_http_parser.py | 46 ++++++++++++++++++++++++++++------ 4 files changed, 53 insertions(+), 20 deletions(-) diff --git a/proxy/common/constants.py b/proxy/common/constants.py index 72efbce57d..46a6c5fb24 100644 --- a/proxy/common/constants.py +++ b/proxy/common/constants.py @@ -8,10 +8,9 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import ipaddress import os import time -import ipaddress - from typing import List from .version import __version__ @@ -73,3 +72,4 @@ DEFAULT_TIMEOUT = 10 DEFAULT_VERSION = False DEFAULT_HTTP_PORT = 80 +DEFAULT_HTTPS_PORT = 443 diff --git a/proxy/http/parser.py b/proxy/http/parser.py index ece1ce0861..6e384ec4a3 100644 --- a/proxy/http/parser.py +++ b/proxy/http/parser.py @@ -8,15 +8,15 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +from typing import Dict, List, NamedTuple, Optional, Tuple, Type, TypeVar from urllib import parse as urlparse -from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List -from .methods import httpMethods -from .chunk_parser import ChunkParser, chunkParserStates - -from ..common.constants import DEFAULT_DISABLE_HEADERS, COLON, CRLF, WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT +from ..common.constants import (COLON, CRLF, DEFAULT_DISABLE_HEADERS, + DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT, + HTTP_1_1, WHITESPACE) from ..common.utils import build_http_request, find_http_line, text_ - +from .chunk_parser import ChunkParser, chunkParserStates +from .methods import httpMethods HttpParserStates = NamedTuple('HttpParserStates', [ ('INITIALIZED', int), @@ -105,14 +105,15 @@ def del_headers(self, headers: List[bytes]) -> None: self.del_header(key.lower()) def set_url(self, url: bytes) -> None: - self.url = urlparse.urlsplit(url) + self.url = urlparse.urlsplit( + (b"//" if self.method == httpMethods.CONNECT else b"") + url) self.set_line_attributes() def set_line_attributes(self) -> None: if self.type == httpParserTypes.REQUEST_PARSER: if self.method == httpMethods.CONNECT and self.url: - u = urlparse.urlsplit(b'//' + self.url.path) - self.host, self.port = u.hostname, u.port + self.host, self.port = self.url.hostname, self.url.port \ + if self.url.port else DEFAULT_HTTPS_PORT elif self.url: self.host, self.port = self.url.hostname, self.url.port \ if self.url.port else DEFAULT_HTTP_PORT @@ -164,7 +165,8 @@ def parse(self, raw: bytes) -> None: self.state = httpParserStates.COMPLETE more = False else: - raise NotImplementedError('Parser shouldn\'t have reached here') + raise NotImplementedError( + 'Parser shouldn\'t have reached here') else: more, raw = self.process(raw) self.buffer = raw diff --git a/requirements-testing.txt b/requirements-testing.txt index 32925741c0..00f933c5fe 100644 --- a/requirements-testing.txt +++ b/requirements-testing.txt @@ -5,7 +5,6 @@ pytest==5.3.2 pytest-cov==2.8.1 autopep8==1.4.4 mypy==0.761 -py-spy==0.3.0 codecov==2.0.15 tox==3.14.3 mccabe==0.6.1 diff --git a/tests/http/test_http_parser.py b/tests/http/test_http_parser.py index 6867749c3b..9564a46d30 100644 --- a/tests/http/test_http_parser.py +++ b/tests/http/test_http_parser.py @@ -11,10 +11,11 @@ import unittest from proxy.common.constants import CRLF -from proxy.common.utils import build_http_request, find_http_line, build_http_response, build_http_header, bytes_ -from proxy.http.methods import httpMethods +from proxy.common.utils import (build_http_header, build_http_request, + build_http_response, bytes_, find_http_line) from proxy.http.codes import httpStatusCodes -from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates +from proxy.http.methods import httpMethods +from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes class TestHttpParser(unittest.TestCase): @@ -123,7 +124,7 @@ def test_get_full_parse(self) -> None: b'Host: %s', CRLF ]) - pkt = raw % (b'https://example.com/path/dir/?a=b&c=d#p=q', + pkt = raw % (b'http://example.com/path/dir/?a=b&c=d#p=q', b'example.com') self.parser.parse(pkt) self.assertEqual(self.parser.total_size, len(pkt)) @@ -134,7 +135,8 @@ def test_get_full_parse(self) -> None: self.assertEqual(self.parser.url.port, None) self.assertEqual(self.parser.version, b'HTTP/1.1') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) - self.assertEqual(self.parser.headers[b'host'], (b'Host', b'example.com')) + self.assertEqual( + self.parser.headers[b'host'], (b'Host', b'example.com')) self.parser.del_headers([b'host']) self.parser.add_headers([(b'Host', b'example.com')]) self.assertEqual( @@ -188,7 +190,8 @@ def test_get_partial_parse1(self) -> None: self.parser.parse(CRLF * 2) self.assertEqual(self.parser.total_size, len(pkt) + (3 * len(CRLF)) + len(host_hdr)) - self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080')) + self.assertEqual( + self.parser.headers[b'host'], (b'Host', b'localhost:8080')) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_get_partial_parse2(self) -> None: @@ -205,7 +208,8 @@ def test_get_partial_parse2(self) -> None: self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) self.parser.parse(b'localhost:8080' + CRLF) - self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080')) + self.assertEqual( + self.parser.headers[b'host'], (b'Host', b'localhost:8080')) self.assertEqual(self.parser.buffer, b'') self.assertEqual( self.parser.state, @@ -524,3 +528,31 @@ def test_paramiko_doc(self) -> None: self.parser = HttpParser(httpParserTypes.RESPONSE_PARSER) self.parser.parse(response) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) + + def test_set_url_http(self) -> None: + self.parser.set_url(b"http://unicorn.fr:8899/path1/path2") + self.assertEqual(self.parser.port, 8899) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2") + + self.parser.set_url(b"http://unicorn.fr/path1/path2") + self.assertEqual(self.parser.port, 80) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2") + + def test_set_url_https(self) -> None: + ''' + The code is setting method as httpMethods.CONNECT when there is HTTPS, + here we set it to just test the set_url methdod + ''' + self.parser.method = httpMethods.CONNECT + + self.parser.set_url(b"unicorn.fr:443/path1/path2") + self.assertEqual(self.parser.port, 443) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2") + + self.parser.set_url(b"unicorn.fr/path1/path2") + self.assertEqual(self.parser.port, 443) + self.assertEqual(self.parser.host, b"unicorn.fr") + self.assertEqual(self.parser.path, b"/path1/path2")