diff --git a/proxy/http/handler.py b/proxy/http/handler.py index e1aff89ee6..c8d070411c 100644 --- a/proxy/http/handler.py +++ b/proxy/http/handler.py @@ -268,9 +268,7 @@ def _discover_plugin_klass(self, protocol: int) -> Optional[Type['HttpProtocolHa def _parse_first_request(self, data: memoryview) -> bool: # Parse http request try: - # TODO(abhinavsingh): Remove .tobytes after parser is - # memoryview compliant - self.request.parse(data.tobytes()) + self.request.parse(data) except HttpProtocolException as e: # noqa: WPS329 self.work.queue(BAD_REQUEST_RESPONSE_PKT) raise e diff --git a/proxy/http/parser/chunk.py b/proxy/http/parser/chunk.py index 691117926d..eb35798955 100644 --- a/proxy/http/parser/chunk.py +++ b/proxy/http/parser/chunk.py @@ -34,13 +34,13 @@ def __init__(self) -> None: # Expected size of next following chunk self.size: Optional[int] = None - def parse(self, raw: bytes) -> bytes: + def parse(self, raw: memoryview) -> memoryview: more = len(raw) > 0 while more and self.state != chunkParserStates.COMPLETE: - more, raw = self.process(raw) + more, raw = self.process(raw.tobytes()) return raw - def process(self, raw: bytes) -> Tuple[bool, bytes]: + def process(self, raw: bytes) -> Tuple[bool, memoryview]: if self.state == chunkParserStates.WAITING_FOR_SIZE: # Consume prior chunk in buffer # in case chunk size without CRLF was received @@ -69,7 +69,7 @@ def process(self, raw: bytes) -> Tuple[bool, bytes]: self.state = chunkParserStates.WAITING_FOR_SIZE self.chunk = b'' self.size = None - return len(raw) > 0, raw + return len(raw) > 0, memoryview(raw) @staticmethod def to_chunks(raw: bytes, chunk_size: int = DEFAULT_BUFFER_SIZE) -> bytes: diff --git a/proxy/http/parser/parser.py b/proxy/http/parser/parser.py index 00790c32d8..7d29f76e40 100644 --- a/proxy/http/parser/parser.py +++ b/proxy/http/parser/parser.py @@ -77,7 +77,7 @@ def __init__( # Total size of raw bytes passed for parsing self.total_size: int = 0 # Buffer to hold unprocessed bytes - self.buffer: bytes = b'' + self.buffer: Optional[memoryview] = None # Internal headers data structure: # - Keys are lower case header names. # - Values are 2-tuple containing original @@ -102,13 +102,13 @@ def request( httpParserTypes.REQUEST_PARSER, enable_proxy_protocol=enable_proxy_protocol, ) - parser.parse(raw) + parser.parse(memoryview(raw)) return parser @classmethod def response(cls: Type[T], raw: bytes) -> T: parser = cls(httpParserTypes.RESPONSE_PARSER) - parser.parse(raw) + parser.parse(memoryview(raw)) return parser def header(self, key: bytes) -> bytes: @@ -206,14 +206,21 @@ def body_expected(self) -> bool: """Returns true if content or chunked response is expected.""" return self._content_expected or self._is_chunked_encoded - def parse(self, raw: bytes, allowed_url_schemes: Optional[List[bytes]] = None) -> None: + def parse( + self, + raw: memoryview, + allowed_url_schemes: Optional[List[bytes]] = None, + ) -> None: """Parses HTTP request out of raw bytes. Check for `HttpParser.state` after `parse` has successfully returned.""" size = len(raw) self.total_size += size - raw = self.buffer + raw - self.buffer, more = b'', size > 0 + if self.buffer: + # TODO(abhinavsingh): Instead of tobytes our parser + # must be capable of working with arrays of memoryview + raw = memoryview(self.buffer.tobytes() + raw.tobytes()) + self.buffer, more = None, size > 0 while more and self.state != httpParserStates.COMPLETE: # gte with HEADERS_COMPLETE also encapsulated RCVING_BODY state if self.state >= httpParserStates.HEADERS_COMPLETE: @@ -237,7 +244,7 @@ def parse(self, raw: bytes, allowed_url_schemes: Optional[List[bytes]] = None) - not (self._content_expected or self._is_chunked_encoded) and \ raw == b'': self.state = httpParserStates.COMPLETE - self.buffer = raw + self.buffer = None if raw == b'' else raw def build(self, disable_headers: Optional[List[bytes]] = None, for_proxy: bool = False) -> bytes: """Rebuild the request object.""" @@ -278,7 +285,7 @@ def build_response(self) -> bytes: body=self._get_body_or_chunks(), ) - def _process_body(self, raw: bytes) -> Tuple[bool, bytes]: + def _process_body(self, raw: memoryview) -> Tuple[bool, memoryview]: # Ref: http://www.ietf.org/rfc/rfc2616.txt # 3.If a Content-Length header field (section 14.13) is present, its # decimal value in OCTETs represents both the entity-length and the @@ -297,7 +304,8 @@ def _process_body(self, raw: bytes) -> Tuple[bool, bytes]: self.body = self.chunk.body self.state = httpParserStates.COMPLETE more = False - elif self._content_expected: + return more, raw + if self._content_expected: self.state = httpParserStates.RCVING_BODY if self.body is None: self.body = b'' @@ -307,23 +315,21 @@ def _process_body(self, raw: bytes) -> Tuple[bool, bytes]: if self.body and \ len(self.body) == int(self.header(b'content-length')): self.state = httpParserStates.COMPLETE - more, raw = len(raw) > 0, raw[total_size - received_size:] - else: - self.state = httpParserStates.RCVING_BODY - # Received a packet without content-length header - # and no transfer-encoding specified. - # - # This can happen for both HTTP/1.0 and HTTP/1.1 scenarios. - # Currently, we consume the remaining buffer as body. - # - # Ref https://github.com/abhinavsingh/proxy.py/issues/398 - # - # See TestHttpParser.test_issue_398 scenario - self.body = raw - more, raw = False, b'' - return more, raw - - def _process_headers(self, raw: bytes) -> Tuple[bool, bytes]: + return len(raw) > 0, raw[total_size - received_size:] + # Received a packet without content-length header + # and no transfer-encoding specified. + # + # This can happen for both HTTP/1.0 and HTTP/1.1 scenarios. + # Currently, we consume the remaining buffer as body. + # + # Ref https://github.com/abhinavsingh/proxy.py/issues/398 + # + # See TestHttpParser.test_issue_398 scenario + self.state = httpParserStates.RCVING_BODY + self.body = raw + return False, memoryview(b'') + + def _process_headers(self, raw: memoryview) -> Tuple[bool, memoryview]: """Returns False when no CRLF could be found in received bytes. TODO: We should not return until parser reaches headers complete @@ -334,10 +340,10 @@ def _process_headers(self, raw: bytes) -> Tuple[bool, bytes]: This will also help make the parser even more stateless. """ while True: - parts = raw.split(CRLF, 1) + parts = raw.tobytes().split(CRLF, 1) if len(parts) == 1: return False, raw - line, raw = parts[0], parts[1] + line, raw = parts[0], memoryview(parts[1]) if self.state in (httpParserStates.LINE_RCVD, httpParserStates.RCVING_HEADERS): if line == b'' or line.strip() == b'': # Blank line received. self.state = httpParserStates.HEADERS_COMPLETE @@ -352,14 +358,14 @@ def _process_headers(self, raw: bytes) -> Tuple[bool, bytes]: def _process_line( self, - raw: bytes, + raw: memoryview, allowed_url_schemes: Optional[List[bytes]] = None, - ) -> Tuple[bool, bytes]: + ) -> Tuple[bool, memoryview]: while True: - parts = raw.split(CRLF, 1) + parts = raw.tobytes().split(CRLF, 1) if len(parts) == 1: return False, raw - line, raw = parts[0], parts[1] + line, raw = parts[0], memoryview(parts[1]) if self.type == httpParserTypes.REQUEST_PARSER: if self.protocol is not None and self.protocol.version is None: # We expect to receive entire proxy protocol v1 line diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index f6ad9a028b..019eb651e7 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -276,11 +276,8 @@ async def read_from_descriptors(self, r: Readables) -> bool: if self.response.is_complete: self.handle_pipeline_response(raw) else: - # TODO(abhinavsingh): Remove .tobytes after parser is - # memoryview compliant - chunk = raw.tobytes() - self.response.parse(chunk) - self.emit_response_events(len(chunk)) + self.response.parse(raw) + self.emit_response_events(len(raw)) else: self.response.total_size += len(raw) # queue raw data for client @@ -430,7 +427,6 @@ def on_client_data(self, raw: memoryview) -> None: # must be treated as WebSocket protocol packets. self.upstream.queue(raw) return - if self.pipeline_request is None: # For pipeline requests, we never # want to use --enable-proxy-protocol flag @@ -443,10 +439,7 @@ def on_client_data(self, raw: memoryview) -> None: self.pipeline_request = HttpParser( httpParserTypes.REQUEST_PARSER, ) - - # TODO(abhinavsingh): Remove .tobytes after parser is - # memoryview compliant - self.pipeline_request.parse(raw.tobytes()) + self.pipeline_request.parse(raw) if self.pipeline_request.is_complete: for plugin in self.plugins.values(): assert self.pipeline_request is not None @@ -555,9 +548,7 @@ def handle_pipeline_response(self, raw: memoryview) -> None: self.pipeline_response = HttpParser( httpParserTypes.RESPONSE_PARSER, ) - # TODO(abhinavsingh): Remove .tobytes after parser is memoryview - # compliant - self.pipeline_response.parse(raw.tobytes()) + self.pipeline_response.parse(raw) if self.pipeline_response.is_complete: self.pipeline_response = None diff --git a/proxy/http/server/web.py b/proxy/http/server/web.py index c3376b3765..06072493b2 100644 --- a/proxy/http/server/web.py +++ b/proxy/http/server/web.py @@ -201,9 +201,7 @@ def on_client_data(self, raw: memoryview) -> None: self.pipeline_request = HttpParser( httpParserTypes.REQUEST_PARSER, ) - # TODO(abhinavsingh): Remove .tobytes after parser is memoryview - # compliant - self.pipeline_request.parse(raw.tobytes()) + self.pipeline_request.parse(raw) if self.pipeline_request.is_complete: self.route.handle_request(self.pipeline_request) if not self.pipeline_request.is_http_1_1_keep_alive: diff --git a/proxy/http/websocket/client.py b/proxy/http/websocket/client.py index 223ff50482..2f61cbab89 100644 --- a/proxy/http/websocket/client.py +++ b/proxy/http/websocket/client.py @@ -77,7 +77,7 @@ def upgrade(self) -> None: ), ) response = HttpParser(httpParserTypes.RESPONSE_PARSER) - response.parse(self.sock.recv(DEFAULT_BUFFER_SIZE)) + response.parse(memoryview(self.sock.recv(DEFAULT_BUFFER_SIZE))) accept = response.header(b'Sec-Websocket-Accept') assert WebsocketFrame.key_to_accept(key) == accept @@ -100,8 +100,6 @@ def run_once(self) -> bool: self.closed = True return True frame = WebsocketFrame() - # TODO(abhinavsingh): Remove .tobytes after parser is - # memoryview compliant frame.parse(raw.tobytes()) self.on_message(frame) elif mask & selectors.EVENT_WRITE: diff --git a/proxy/plugin/modify_chunk_response.py b/proxy/plugin/modify_chunk_response.py index 16171e1f11..f050121fc0 100644 --- a/proxy/plugin/modify_chunk_response.py +++ b/proxy/plugin/modify_chunk_response.py @@ -32,7 +32,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: def handle_upstream_chunk(self, chunk: memoryview) -> Optional[memoryview]: # Parse the response. # Note that these chunks also include headers - self.response.parse(chunk.tobytes()) + self.response.parse(chunk) # If response is complete, modify and dispatch to client if self.response.is_complete: # Avoid setting a body for responses where a body is not expected. diff --git a/tests/http/parser/test_chunk_parser.py b/tests/http/parser/test_chunk_parser.py index 1fde17256a..5df67264f1 100644 --- a/tests/http/parser/test_chunk_parser.py +++ b/tests/http/parser/test_chunk_parser.py @@ -20,16 +20,18 @@ def setUp(self) -> None: def test_chunk_parse_basic(self) -> None: self.parser.parse( - b''.join([ - b'4\r\n', - b'Wiki\r\n', - b'5\r\n', - b'pedia\r\n', - b'E\r\n', - b' in\r\n\r\nchunks.\r\n', - b'0\r\n', - b'\r\n', - ]), + memoryview( + b''.join([ + b'4\r\n', + b'Wiki\r\n', + b'5\r\n', + b'pedia\r\n', + b'E\r\n', + b' in\r\n\r\nchunks.\r\n', + b'0\r\n', + b'\r\n', + ]), + ), ) self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) @@ -38,7 +40,7 @@ def test_chunk_parse_basic(self) -> None: def test_chunk_parse_issue_27(self) -> None: """Case when data ends with the chunk size but without ending CRLF.""" - self.parser.parse(b'3') + self.parser.parse(memoryview(b'3')) self.assertEqual(self.parser.chunk, b'3') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'') @@ -46,7 +48,7 @@ def test_chunk_parse_issue_27(self) -> None: self.parser.state, chunkParserStates.WAITING_FOR_SIZE, ) - self.parser.parse(b'\r\n') + self.parser.parse(memoryview(b'\r\n')) self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, 3) self.assertEqual(self.parser.body, b'') @@ -54,7 +56,7 @@ def test_chunk_parse_issue_27(self) -> None: self.parser.state, chunkParserStates.WAITING_FOR_DATA, ) - self.parser.parse(b'abc') + self.parser.parse(memoryview(b'abc')) self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'abc') @@ -62,7 +64,7 @@ def test_chunk_parse_issue_27(self) -> None: self.parser.state, chunkParserStates.WAITING_FOR_SIZE, ) - self.parser.parse(b'\r\n') + self.parser.parse(memoryview(b'\r\n')) self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'abc') @@ -70,7 +72,7 @@ def test_chunk_parse_issue_27(self) -> None: self.parser.state, chunkParserStates.WAITING_FOR_SIZE, ) - self.parser.parse(b'4\r\n') + self.parser.parse(memoryview(b'4\r\n')) self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, 4) self.assertEqual(self.parser.body, b'abc') @@ -78,7 +80,7 @@ def test_chunk_parse_issue_27(self) -> None: self.parser.state, chunkParserStates.WAITING_FOR_DATA, ) - self.parser.parse(b'defg\r\n0') + self.parser.parse(memoryview(b'defg\r\n0')) self.assertEqual(self.parser.chunk, b'0') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'abcdefg') @@ -86,7 +88,7 @@ def test_chunk_parse_issue_27(self) -> None: self.parser.state, chunkParserStates.WAITING_FOR_SIZE, ) - self.parser.parse(b'\r\n\r\n') + self.parser.parse(memoryview(b'\r\n\r\n')) self.assertEqual(self.parser.chunk, b'') self.assertEqual(self.parser.size, None) self.assertEqual(self.parser.body, b'abcdefg') diff --git a/tests/http/parser/test_http_parser.py b/tests/http/parser/test_http_parser.py index a8192c8a8e..95c052dec4 100644 --- a/tests/http/parser/test_http_parser.py +++ b/tests/http/parser/test_http_parser.py @@ -28,37 +28,39 @@ def setUp(self) -> None: def test_issue_127(self) -> None: with self.assertRaises(HttpProtocolException): - self.parser.parse(CRLF) + self.parser.parse(memoryview(CRLF)) with self.assertRaises(HttpProtocolException): raw = b'qwqrqw!@!#@!#ad adfad\r\n' while True: - self.parser.parse(raw) + self.parser.parse(memoryview(raw)) def test_issue_398(self) -> None: p = HttpParser(httpParserTypes.RESPONSE_PARSER) - p.parse(HTTP_1_0 + b' 200 OK' + CRLF) + p.parse(memoryview(HTTP_1_0 + b' 200 OK' + CRLF)) self.assertEqual(p.version, HTTP_1_0) self.assertEqual(p.code, b'200') self.assertEqual(p.reason, b'OK') self.assertEqual(p.state, httpParserStates.LINE_RCVD) p.parse( - b'CP=CAO PSA OUR' + CRLF + - b'Cache-Control:private,max-age=0;' + CRLF + - b'X-Frame-Options:SAMEORIGIN' + CRLF + - b'X-Content-Type-Options:nosniff' + CRLF + - b'X-XSS-Protection:1; mode=block' + CRLF + - b'Content-Security-Policy:default-src \'self\' \'unsafe-inline\' \'unsafe-eval\'' + CRLF + - b'Strict-Transport-Security:max-age=2592000; includeSubdomains' + CRLF + - b'Set-Cookie: lang=eng; path=/;HttpOnly;' + CRLF + - b'Content-type:text/html;charset=UTF-8;' + CRLF + CRLF + - b'', + memoryview( + b'CP=CAO PSA OUR' + CRLF + + b'Cache-Control:private,max-age=0;' + CRLF + + b'X-Frame-Options:SAMEORIGIN' + CRLF + + b'X-Content-Type-Options:nosniff' + CRLF + + b'X-XSS-Protection:1; mode=block' + CRLF + + b'Content-Security-Policy:default-src \'self\' \'unsafe-inline\' \'unsafe-eval\'' + CRLF + + b'Strict-Transport-Security:max-age=2592000; includeSubdomains' + CRLF + + b'Set-Cookie: lang=eng; path=/;HttpOnly;' + CRLF + + b'Content-type:text/html;charset=UTF-8;' + CRLF + CRLF + + b'', + ), ) self.assertEqual(p.body, b'') self.assertEqual(p.state, httpParserStates.RCVING_BODY) def test_urlparse(self) -> None: - self.parser.parse(b'CONNECT httpbin.org:443 HTTP/1.1\r\n') + self.parser.parse(memoryview(b'CONNECT httpbin.org:443 HTTP/1.1\r\n')) self.assertTrue(self.parser.is_https_tunnel) self.assertFalse(self.parser.is_connection_upgrade) self.assertTrue(self.parser.is_http_1_1_keep_alive) @@ -69,40 +71,52 @@ def test_urlparse(self) -> None: self.assertNotEqual(self.parser.state, httpParserStates.COMPLETE) def test_urlparse_on_invalid_connect_request(self) -> None: - self.parser.parse(b'CONNECT / HTTP/1.0\r\n\r\n') + self.parser.parse(memoryview(b'CONNECT / HTTP/1.0\r\n\r\n')) self.assertTrue(self.parser.is_https_tunnel) self.assertEqual(self.parser.host, None) self.assertEqual(self.parser.port, 443) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_unicode_character_domain_connect(self) -> None: - self.parser.parse(bytes_('CONNECT ççç.org:443 HTTP/1.1\r\n')) + self.parser.parse( + memoryview( + bytes_('CONNECT ççç.org:443 HTTP/1.1\r\n'), + ), + ) self.assertTrue(self.parser.is_https_tunnel) self.assertEqual(self.parser.host, bytes_('ççç.org')) self.assertEqual(self.parser.port, 443) def test_invalid_ipv6_in_request_line(self) -> None: self.parser.parse( - bytes_('CONNECT 2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF:443 HTTP/1.1\r\n'), + memoryview( + bytes_('CONNECT 2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF:443 HTTP/1.1\r\n'), + ), ) self.assertTrue(self.parser.is_https_tunnel) self.assertEqual( - self.parser.host, bytes_( - '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', + self.parser.host, memoryview( + bytes_( + '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', + ), ), ) self.assertEqual(self.parser.port, 443) def test_valid_ipv6_in_request_line(self) -> None: self.parser.parse( - bytes_( - 'CONNECT [2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:443 HTTP/1.1\r\n', + memoryview( + bytes_( + 'CONNECT [2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]:443 HTTP/1.1\r\n', + ), ), ) self.assertTrue(self.parser.is_https_tunnel) self.assertEqual( - self.parser.host, bytes_( - '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', + self.parser.host, memoryview( + bytes_( + '[2001:db8:3333:4444:CCCC:DDDD:EEEE:FFFF]', + ), ), ) self.assertEqual(self.parser.port, 443) @@ -223,9 +237,9 @@ def test_find_line_returns_None(self) -> None: def test_connect_request_with_crlf_as_separate_chunk(self) -> None: """See https://github.com/abhinavsingh/py/issues/70 for background.""" raw = b'CONNECT pypi.org:443 HTTP/1.0\r\n' - self.parser.parse(raw) + self.parser.parse(memoryview(raw)) self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) - self.parser.parse(CRLF) + self.parser.parse(memoryview(CRLF)) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_get_full_parse(self) -> None: @@ -238,7 +252,7 @@ def test_get_full_parse(self) -> None: b'https://example.com/path/dir/?a=b&c=d#p=q', b'example.com', ) - self.parser.parse(pkt) + self.parser.parse(memoryview(pkt)) self.assertEqual(self.parser.total_size, len(pkt)) assert self.parser._url and self.parser._url.remainder self.assertEqual(self.parser._url.remainder, b'/path/dir/?a=b&c=d#p=q') @@ -264,7 +278,7 @@ def test_get_full_parse(self) -> None: def test_line_rcvd_to_rcving_headers_state_change(self) -> None: pkt = b'GET http://localhost HTTP/1.1' - self.parser.parse(pkt) + self.parser.parse(memoryview(pkt)) self.assertEqual(self.parser.total_size, len(pkt)) self.assert_state_change_with_crlf( httpParserStates.INITIALIZED, @@ -276,7 +290,7 @@ def test_get_partial_parse1(self) -> None: pkt = CRLF.join([ b'GET http://localhost:8080 HTTP/1.1', ]) - self.parser.parse(pkt) + self.parser.parse(memoryview(pkt)) self.assertEqual(self.parser.total_size, len(pkt)) self.assertEqual(self.parser.method, None) self.assertEqual(self.parser._url, None) @@ -286,7 +300,7 @@ def test_get_partial_parse1(self) -> None: httpParserStates.INITIALIZED, ) - self.parser.parse(CRLF) + self.parser.parse(memoryview(CRLF)) self.assertEqual(self.parser.total_size, len(pkt) + len(CRLF)) self.assertEqual(self.parser.method, b'GET') assert self.parser._url @@ -296,7 +310,7 @@ def test_get_partial_parse1(self) -> None: self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) host_hdr = b'Host: localhost:8080' - self.parser.parse(host_hdr) + self.parser.parse(memoryview(host_hdr)) self.assertEqual( self.parser.total_size, len(pkt) + len(CRLF) + len(host_hdr), @@ -305,7 +319,7 @@ def test_get_partial_parse1(self) -> None: self.assertEqual(self.parser.buffer, b'Host: localhost:8080') self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) - self.parser.parse(CRLF * 2) + self.parser.parse(memoryview(CRLF * 2)) self.assertEqual( self.parser.total_size, len(pkt) + (3 * len(CRLF)) + len(host_hdr), @@ -322,10 +336,12 @@ def test_get_partial_parse1(self) -> None: def test_get_partial_parse2(self) -> None: self.parser.parse( - CRLF.join([ - b'GET http://localhost:8080 HTTP/1.1', - b'Host: ', - ]), + memoryview( + CRLF.join([ + b'GET http://localhost:8080 HTTP/1.1', + b'Host: ', + ]), + ), ) self.assertEqual(self.parser.method, b'GET') assert self.parser._url @@ -335,7 +351,7 @@ def test_get_partial_parse2(self) -> None: self.assertEqual(self.parser.buffer, b'Host: ') self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) - self.parser.parse(b'localhost:8080' + CRLF) + self.parser.parse(memoryview(b'localhost:8080' + CRLF)) assert self.parser.headers self.assertEqual( self.parser.headers[b'host'], @@ -344,14 +360,14 @@ def test_get_partial_parse2(self) -> None: b'localhost:8080', ), ) - self.assertEqual(self.parser.buffer, b'') + self.assertEqual(self.parser.buffer, None) self.assertEqual( self.parser.state, httpParserStates.RCVING_HEADERS, ) - self.parser.parse(b'Content-Type: text/plain' + CRLF) - self.assertEqual(self.parser.buffer, b'') + self.parser.parse(memoryview(b'Content-Type: text/plain' + CRLF)) + self.assertEqual(self.parser.buffer, None) assert self.parser.headers self.assertEqual( self.parser.headers[b'content-type'], ( @@ -364,7 +380,7 @@ def test_get_partial_parse2(self) -> None: httpParserStates.RCVING_HEADERS, ) - self.parser.parse(CRLF) + self.parser.parse(memoryview(CRLF)) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) def test_post_full_parse(self) -> None: @@ -375,7 +391,7 @@ def test_post_full_parse(self) -> None: b'Content-Type: application/x-www-form-urlencoded' + CRLF, b'a=b&c=d', ]) - self.parser.parse(raw % b'http://localhost') + self.parser.parse(memoryview(raw % b'http://localhost')) self.assertEqual(self.parser.method, b'POST') assert self.parser._url self.assertEqual(self.parser._url.hostname, b'localhost') @@ -391,7 +407,7 @@ def test_post_full_parse(self) -> None: (b'Content-Length', b'7'), ) self.assertEqual(self.parser.body, b'a=b&c=d') - self.assertEqual(self.parser.buffer, b'') + self.assertEqual(self.parser.buffer, None) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) self.assertEqual(len(self.parser.build()), len(raw % b'/')) @@ -402,19 +418,21 @@ def assert_state_change_with_crlf( final_state: int, ) -> None: self.assertEqual(self.parser.state, initial_state) - self.parser.parse(CRLF) + self.parser.parse(memoryview(CRLF)) self.assertEqual(self.parser.state, next_state) - self.parser.parse(CRLF) + self.parser.parse(memoryview(CRLF)) self.assertEqual(self.parser.state, final_state) def test_post_partial_parse(self) -> None: self.parser.parse( - CRLF.join([ - b'POST http://localhost HTTP/1.1', - b'Host: localhost', - b'Content-Length: 7', - b'Content-Type: application/x-www-form-urlencoded', - ]), + memoryview( + CRLF.join([ + b'POST http://localhost HTTP/1.1', + b'Host: localhost', + b'Content-Length: 7', + b'Content-Type: application/x-www-form-urlencoded', + ]), + ), ) self.assertEqual(self.parser.method, b'POST') assert self.parser._url @@ -427,18 +445,18 @@ def test_post_partial_parse(self) -> None: httpParserStates.HEADERS_COMPLETE, ) - self.parser.parse(b'a=b') + self.parser.parse(memoryview(b'a=b')) self.assertEqual( self.parser.state, httpParserStates.RCVING_BODY, ) self.assertEqual(self.parser.body, b'a=b') - self.assertEqual(self.parser.buffer, b'') + self.assertEqual(self.parser.buffer, None) - self.parser.parse(b'&c=d') + self.parser.parse(memoryview(b'&c=d')) self.assertEqual(self.parser.state, httpParserStates.COMPLETE) self.assertEqual(self.parser.body, b'a=b&c=d') - self.assertEqual(self.parser.buffer, b'') + self.assertEqual(self.parser.buffer, None) def test_connect_request_without_host_header_request_parse(self) -> None: """Case where clients can send CONNECT request without a Host header field. @@ -451,7 +469,7 @@ def test_connect_request_without_host_header_request_parse(self) -> None: See https://github.com/abhinavsingh/py/issues/5 for details. """ - self.parser.parse(b'CONNECT pypi.org:443 HTTP/1.0\r\n\r\n') + self.parser.parse(memoryview(b'CONNECT pypi.org:443 HTTP/1.0\r\n\r\n')) self.assertEqual(self.parser.method, httpMethods.CONNECT) self.assertEqual(self.parser.version, b'HTTP/1.0') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) @@ -466,12 +484,14 @@ def test_request_parse_without_content_length(self) -> None: See https://github.com/abhinavsingh/py/issues/20 for details. """ self.parser.parse( - CRLF.join([ - b'POST http://localhost HTTP/1.1', - b'Host: localhost', - b'Content-Type: application/x-www-form-urlencoded', - CRLF, - ]), + memoryview( + CRLF.join([ + b'POST http://localhost HTTP/1.1', + b'Host: localhost', + b'Content-Type: application/x-www-form-urlencoded', + CRLF, + ]), + ), ) self.assertEqual(self.parser.method, b'POST') self.assertEqual(self.parser.state, httpParserStates.COMPLETE) @@ -492,16 +512,18 @@ def test_response_parse_without_content_length(self) -> None: pipelined responses not trigger stream close but may receive multiple responses. """ self.parser.type = httpParserTypes.RESPONSE_PARSER - self.parser.parse(b'HTTP/1.0 200 OK' + CRLF) + self.parser.parse(memoryview(b'HTTP/1.0 200 OK' + CRLF)) self.assertEqual(self.parser.code, b'200') self.assertEqual(self.parser.version, b'HTTP/1.0') self.assertEqual(self.parser.state, httpParserStates.LINE_RCVD) self.parser.parse( - CRLF.join([ - b'Server: BaseHTTP/0.3 Python/2.7.10', - b'Date: Thu, 13 Dec 2018 16:24:09 GMT', - CRLF, - ]), + memoryview( + CRLF.join([ + b'Server: BaseHTTP/0.3 Python/2.7.10', + b'Date: Thu, 13 Dec 2018 16:24:09 GMT', + CRLF, + ]), + ), ) self.assertEqual( self.parser.state, @@ -511,22 +533,24 @@ def test_response_parse_without_content_length(self) -> None: def test_response_parse(self) -> None: self.parser.type = httpParserTypes.RESPONSE_PARSER self.parser.parse( - b''.join([ - b'HTTP/1.1 301 Moved Permanently\r\n', - b'Location: http://www.google.com/\r\n', - b'Content-Type: text/html; charset=UTF-8\r\n', - b'Date: Wed, 22 May 2013 14:07:29 GMT\r\n', - b'Expires: Fri, 21 Jun 2013 14:07:29 GMT\r\n', - b'Cache-Control: public, max-age=2592000\r\n', - b'Server: gws\r\n', - b'Content-Length: 219\r\n', - b'X-XSS-Protection: 1; mode=block\r\n', - b'X-Frame-Options: SAMEORIGIN\r\n\r\n', - b'
\n' + - b'