Skip to content

Fix for urllib with python 3.7+ & 3.8+ #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Fix for urllib with python 3.7+ & 3.8+ #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions proxy/common/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
import ipaddress
import os
import time
import ipaddress

from typing import List

from .version import __version__
Expand Down Expand Up @@ -73,3 +72,4 @@
DEFAULT_TIMEOUT = 10
DEFAULT_VERSION = False
DEFAULT_HTTP_PORT = 80
DEFAULT_HTTPS_PORT = 443
22 changes: 12 additions & 10 deletions proxy/http/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@
:copyright: (c) 2013-present by Abhinav Singh and contributors.
:license: BSD, see LICENSE for more details.
"""
from typing import Dict, List, NamedTuple, Optional, Tuple, Type, TypeVar
from urllib import parse as urlparse
from typing import TypeVar, NamedTuple, Optional, Dict, Type, Tuple, List

from .methods import httpMethods
from .chunk_parser import ChunkParser, chunkParserStates

from ..common.constants import DEFAULT_DISABLE_HEADERS, COLON, CRLF, WHITESPACE, HTTP_1_1, DEFAULT_HTTP_PORT
from ..common.constants import (COLON, CRLF, DEFAULT_DISABLE_HEADERS,
DEFAULT_HTTP_PORT, DEFAULT_HTTPS_PORT,
HTTP_1_1, WHITESPACE)
from ..common.utils import build_http_request, find_http_line, text_

from .chunk_parser import ChunkParser, chunkParserStates
from .methods import httpMethods

HttpParserStates = NamedTuple('HttpParserStates', [
('INITIALIZED', int),
Expand Down Expand Up @@ -105,14 +105,15 @@ def del_headers(self, headers: List[bytes]) -> None:
self.del_header(key.lower())

def set_url(self, url: bytes) -> None:
self.url = urlparse.urlsplit(url)
self.url = urlparse.urlsplit(
(b"//" if self.method == httpMethods.CONNECT else b"") + url)
self.set_line_attributes()

def set_line_attributes(self) -> None:
if self.type == httpParserTypes.REQUEST_PARSER:
if self.method == httpMethods.CONNECT and self.url:
u = urlparse.urlsplit(b'//' + self.url.path)
self.host, self.port = u.hostname, u.port
self.host, self.port = self.url.hostname, self.url.port \
if self.url.port else DEFAULT_HTTPS_PORT
elif self.url:
self.host, self.port = self.url.hostname, self.url.port \
if self.url.port else DEFAULT_HTTP_PORT
Expand Down Expand Up @@ -164,7 +165,8 @@ def parse(self, raw: bytes) -> None:
self.state = httpParserStates.COMPLETE
more = False
else:
raise NotImplementedError('Parser shouldn\'t have reached here')
raise NotImplementedError(
'Parser shouldn\'t have reached here')
else:
more, raw = self.process(raw)
self.buffer = raw
Expand Down
1 change: 0 additions & 1 deletion requirements-testing.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ pytest==5.3.2
pytest-cov==2.8.1
autopep8==1.4.4
mypy==0.761
py-spy==0.3.0
Copy link
Contributor Author

@Benouare Benouare Jan 23, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, if we don't drop this, the pip install -r will not work :
Could not find a version that satisfies the requirement py-spy==0.3.0

codecov==2.0.15
tox==3.14.3
mccabe==0.6.1
46 changes: 39 additions & 7 deletions tests/http/test_http_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,11 @@
import unittest

from proxy.common.constants import CRLF
from proxy.common.utils import build_http_request, find_http_line, build_http_response, build_http_header, bytes_
from proxy.http.methods import httpMethods
from proxy.common.utils import (build_http_header, build_http_request,
build_http_response, bytes_, find_http_line)
from proxy.http.codes import httpStatusCodes
from proxy.http.parser import HttpParser, httpParserTypes, httpParserStates
from proxy.http.methods import httpMethods
from proxy.http.parser import HttpParser, httpParserStates, httpParserTypes


class TestHttpParser(unittest.TestCase):
Expand Down Expand Up @@ -123,7 +124,7 @@ def test_get_full_parse(self) -> None:
b'Host: %s',
CRLF
])
pkt = raw % (b'https://example.com/path/dir/?a=b&c=d#p=q',
pkt = raw % (b'http://example.com/path/dir/?a=b&c=d#p=q',
b'example.com')
self.parser.parse(pkt)
self.assertEqual(self.parser.total_size, len(pkt))
Expand All @@ -134,7 +135,8 @@ def test_get_full_parse(self) -> None:
self.assertEqual(self.parser.url.port, None)
self.assertEqual(self.parser.version, b'HTTP/1.1')
self.assertEqual(self.parser.state, httpParserStates.COMPLETE)
self.assertEqual(self.parser.headers[b'host'], (b'Host', b'example.com'))
self.assertEqual(
self.parser.headers[b'host'], (b'Host', b'example.com'))
self.parser.del_headers([b'host'])
self.parser.add_headers([(b'Host', b'example.com')])
self.assertEqual(
Expand Down Expand Up @@ -188,7 +190,8 @@ def test_get_partial_parse1(self) -> None:
self.parser.parse(CRLF * 2)
self.assertEqual(self.parser.total_size, len(pkt) +
(3 * len(CRLF)) + len(host_hdr))
self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
self.assertEqual(
self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
self.assertEqual(self.parser.state, httpParserStates.COMPLETE)

def test_get_partial_parse2(self) -> None:
Expand All @@ -205,7 +208,8 @@ def test_get_partial_parse2(self) -> None:
self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD)

self.parser.parse(b'localhost:8080' + CRLF)
self.assertEqual(self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
self.assertEqual(
self.parser.headers[b'host'], (b'Host', b'localhost:8080'))
self.assertEqual(self.parser.buffer, b'')
self.assertEqual(
self.parser.state,
Expand Down Expand Up @@ -524,3 +528,31 @@ def test_paramiko_doc(self) -> None:
self.parser = HttpParser(httpParserTypes.RESPONSE_PARSER)
self.parser.parse(response)
self.assertEqual(self.parser.state, httpParserStates.COMPLETE)

def test_set_url_http(self) -> None:
self.parser.set_url(b"http://unicorn.fr:8899/path1/path2")
self.assertEqual(self.parser.port, 8899)
self.assertEqual(self.parser.host, b"unicorn.fr")
self.assertEqual(self.parser.path, b"/path1/path2")

self.parser.set_url(b"http://unicorn.fr/path1/path2")
self.assertEqual(self.parser.port, 80)
self.assertEqual(self.parser.host, b"unicorn.fr")
self.assertEqual(self.parser.path, b"/path1/path2")

def test_set_url_https(self) -> None:
'''
The code is setting method as httpMethods.CONNECT when there is HTTPS,
here we set it to just test the set_url methdod
'''
self.parser.method = httpMethods.CONNECT

self.parser.set_url(b"unicorn.fr:443/path1/path2")
self.assertEqual(self.parser.port, 443)
self.assertEqual(self.parser.host, b"unicorn.fr")
self.assertEqual(self.parser.path, b"/path1/path2")

self.parser.set_url(b"unicorn.fr/path1/path2")
self.assertEqual(self.parser.port, 443)
self.assertEqual(self.parser.host, b"unicorn.fr")
self.assertEqual(self.parser.path, b"/path1/path2")