From 7c2b0777ddb64d5e2882aec056bbe47f14210a88 Mon Sep 17 00:00:00 2001 From: Jerry Date: Fri, 19 Nov 2021 16:35:54 +0800 Subject: [PATCH 01/14] add connection status check in read_from_descriptors add connection status check in read_from_descriptors --- proxy/http/proxy/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 6f76e94eaa..c95dba1ba2 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -298,8 +298,11 @@ def read_from_descriptors(self, r: Readables) -> bool: return self._close_and_release() if raw is None: - logger.debug('Server closed connection, tearing down...') - return self._close_and_release() + if self.upstream.closed: + logger.debug('Server closed connection, tearing down...') + return self._close_and_release() + else: + return False for plugin in self.plugins.values(): raw = plugin.handle_upstream_chunk(raw) From ad1eee8fdb2d8ef83eb78a4669f9cc1d3a5bd7ae Mon Sep 17 00:00:00 2001 From: Jerry Date: Fri, 19 Nov 2021 16:38:52 +0800 Subject: [PATCH 02/14] add tls parser skeleton add tls parser skeleton --- proxy/http/parser/tls.py | 310 ++++++++++++++++++++++++++++++++++ tests/http/test_tls_parser.py | 44 +++++ 2 files changed, 354 insertions(+) create mode 100644 proxy/http/parser/tls.py create mode 100644 tests/http/test_tls_parser.py diff --git a/proxy/http/parser/tls.py b/proxy/http/parser/tls.py new file mode 100644 index 0000000000..6bee1876d7 --- /dev/null +++ b/proxy/http/parser/tls.py @@ -0,0 +1,310 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import struct +import logging + +from typing import NamedTuple, Optional + + +logger = logging.getLogger(__name__) + + +TlsContentType = NamedTuple( + 'TlsContentType', [ + ('CHANGE_CIPHER_SPEC', int), + ('ALERT', int), + ('HANDSHAKE', int), + ('APPLICATION_DATA', int), + ('OTHER', int), + ], + ) +tlsContentType = TlsContentType(20, 21, 22, 23, 255) + +class TlsProtocolVersion: + """Protocol Version""" + + def __init__(self) -> None: + self.major = 0 + self.minor = 0 + + def set_value(major: int, minor: int) -> None: + self.major = major + self.minor = minor + + +TlsHandshakeType = NamedTuple( + 'TlsHandshakeType', [ + ('HELLO_REQUEST', int), + ('CLIENT_HELLO', int), + ('SERVER_HELLO', int), + ('CERTIFICATE', int), + ('SERVER_KEY_EXCHANGE', int), + ('CERTIFICATE_REQUEST', int), + ('SERVER_HELLO_DONE', int), + ('CERTIFICATE_VERIFY', int), + ('CLIENT_KEY_EXCHANGE', int), + ('FINISHED', int), + ('OTHER', int), + ], + ) +tlsHandshakeType = TlsHandshakeType(0, 1, 2, 11, 12, 13, 14, 15, 16, 20, 255) + +class TlsHelloRequest: + """TLS Hello Request""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsClientHello: + """TLS Client Hello""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsServerHello: + """TLS Server Hello""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsCertificate: + """TLS Certificate""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsServerKeyExchange: + """TLS Server Key Exchange""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsCertificateRequest: + """TLS Certificate Request""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsServerHelloDone: + """TLS Server Hello Done""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsCertificateVerify: + """TLS Certificate Vefiry""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsClientKeyExchange: + """TLS Client Key Exchange""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsFinished: + """TLS Finished""" + + def __init__(self) -> None: + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + self.data = raw + + def build(self) -> bytes: + return self.data + + +class TlsHandshake: + """TLS Handshake""" + + def __init__(self) -> None: + self.handshake_type: int = tlsHandshakeType.OTHER + self.length: int = 0 + self.hello_request: Optional[TlsHelloRequest] = None + self.client_hellp: Optional[TlsClientHello] = None + self.server_hello: Optional[TlsServerHello] = None + self.certificate: Optional[TlsCertificate] = None + self.server_key_exchange: Optional[TlsServerKeyExchange] = None + self.certificate_request: Optional[TlsCertificateRequest] = None + self.server_hello_done: Optional[TlsServerHelloDone] = None + self.certificate_verify: Optional[TlsCertificateVerify] = None + self.client_key_exchange: Optional[TlsClientKeyExchange] = None + self.finished: Optional[TlsFinished] = None + self.data: Optional[bytes] = None + + def parse(self, raw: bytes) -> bytes: + length = len(raw) + if length < 4: + logger.debug('invalid data, len(raw) = %s', legth) + return raw + else: + payload_length, = struct.unpack('!H', b'\x00' + raw[1:4]) + self.length = payload_length + if length < 4 + payload_length: + logger.debug('incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length) + return raw + else: + # parse + self.handshake_type = raw[0] + self.data = raw[: 4 + payload_length] + payload = raw[4: 4 + payload_length] + if self.handshake_type == tlsHandshakeType.HELLO_REQUEST: + # parse hello request + self.hello_request = TlsHelloRequest() + self.hello_request.parse(payload) + elif self.handshake_type == tlsHandshakeType.CLIENT_HELLO: + # parse client hello + self.client_hellp = TlsClientHello() + self.client_hellp.parse(payload) + elif self.handshake_type == tlsHandshakeType.SERVER_HELLO: + # parse server hello + self.server_hello = TlsServerHello() + self.server_hello.parse(payload) + elif self.handshake_type == tlsHandshakeType.CERTIFICATE: + # parse certficate + self.certificate = TlsCertificate() + self.certificate.parse(payload) + elif self.handshake_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: + # parse server key exchange + self.server_key_exchange = TlsServerKeyExchange() + self.server_key_exchange.parse(payload) + elif self.handshake_type == tlsHandshakeType.CERTIFICATE_REQUEST: + # parse certificate request + self.certificate_request = TlsCertificateRequest() + self.certificate_request.parse(payload) + elif self.handshake_type == tlsHandshakeType.SERVER_HELLO_DONE: + # parse server hello done + self.server_hello_done = TlsServerHelloDone() + self.server_hello_done.parse(payload) + elif self.handshake_type == tlsHandshakeType.CERTIFICATE_VERIFY: + # parse certificate verify + self.certificate_verify = TlsCertificateVerify() + self.certificate_verify.parse(payload) + elif self.handshake_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE: + # parse client key exchange + self.client_key_exchange = TlsClientKeyExchange() + self.client_key_exchange.parse(payload) + elif self.handshake_type == tlsHandshakeType.FINISHED: + # parse finished + self.finished = TlsFinished() + self.finished.parse(payload) + return raw[4 + payload_length:] + + def build(self) -> bytes: + return self.data + + +class TlsParser: + """TLS packet parser""" + + def __init__(self) -> None: + self.content_type: int = tlsContentType.OTHER + self.protocol_version: Optional[TlsProtocolVersion] = None + self.length: int = 0 + self.data: Optional[bytes] = None + # only parse hand shake payload temporary + self.handshake: Optional[TlsHandshake] = None + + def parse(self, raw: bytes) -> bytes: + """parse TLS fragmentation + Ref: https://datatracker.ietf.org/doc/html/rfc5246#page-15 + """ + length = len(raw) + if length < 5: + logger.debug('invalid data, len(raw) = %s', length) + return raw + else: + payload_length, = struct.unpack('!H', raw[3:5]) + self.length = payload_length + if length < 5 + payload_length: + logger.debug('incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length) + return raw + else: + # parse + self.content_type = raw[0] + self.data = raw[: 5 + payload_length] + payload = raw[5:5 + payload_length] + if self.content_type == tlsContentType.HANDSHAKE: + # parse handshake + self.handshake = TlsHandshake() + self.handshake.parse(payload) + self.data = raw[:5 + payload_length] + return raw[5 + payload_length:] + + def build(self) -> bytes: + return self.data + + diff --git a/tests/http/test_tls_parser.py b/tests/http/test_tls_parser.py new file mode 100644 index 0000000000..8fd7b43380 --- /dev/null +++ b/tests/http/test_tls_parser.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +""" + proxy.py + ~~~~~~~~ + ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on + Network monitoring, controls & Application development, testing, debugging. + + :copyright: (c) 2013-present by Abhinav Singh and contributors. + :license: BSD, see LICENSE for more details. +""" +import unittest +import binascii +import re + +from proxy.http.parser.tls import TlsParser, TlsHandshake +from proxy.http.parser.tls import tlsContentType, tlsHandshakeType + + +class TestTlsParser(unittest.TestCase): + """ + Ref: https://tls.ulfheim.net/ + """ + + def unhexlify(self, raw: str) -> bytes: + return binascii.unhexlify(re.sub(r'\s', '', raw)) + + def test_parse_client_hello(self) -> None: + data =""" + 16 03 01 00 a5 01 00 00 a1 03 03 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 + 15 16 17 18 19 1a 1b 1c 1d 1e 1f 00 00 20 cc a8 cc a9 c0 2f c0 30 c0 2b c0 2c c0 13 c0 09 c0 14 + c0 0a 00 9c 00 9d 00 2f 00 35 c0 12 00 0a 01 00 00 58 00 00 00 18 00 16 00 00 13 65 78 61 6d 70 + 6c 65 2e 75 6c 66 68 65 69 6d 2e 6e 65 74 00 05 00 05 01 00 00 00 00 00 0a 00 0a 00 08 00 1d 00 + 17 00 18 00 19 00 0b 00 02 01 00 00 0d 00 12 00 10 04 01 04 03 05 01 05 03 06 01 06 03 02 01 02 + 03 ff 01 00 01 00 00 12 00 00 + """ + raw = b'\x81\x85\xc6\ti\x8d\xael\x05\xe1\xa9' + tls = TlsParser() + tls.parse(self.unhexlify(data)) + self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE) + self.assertEqual(tls.length, 0xa5) + self.assertEqual(tls.handshake.handshake_type, tlsHandshakeType.CLIENT_HELLO) + self.assertEqual(tls.handshake.length, 0xa1) + self.assertEqual(len(tls.handshake.build()), 0xa1 + 0x04) + From bc1622358f1907e0b51e2fb22e812073f54828b1 Mon Sep 17 00:00:00 2001 From: Jerry Date: Fri, 10 Dec 2021 09:27:22 +0800 Subject: [PATCH 03/14] temporary commit temporary commit --- proxy/core/connection/connection.py | 2 + proxy/core/connection/pool.py | 7 +- proxy/core/connection/server.py | 4 +- proxy/http/parser/tls.py | 229 +++++++++++++++++++++------- proxy/http/proxy/server.py | 57 ++++++- tests/http/test_tls_parser.py | 20 ++- 6 files changed, 252 insertions(+), 67 deletions(-) diff --git a/proxy/core/connection/connection.py b/proxy/core/connection/connection.py index 65b01769bd..89f3108861 100644 --- a/proxy/core/connection/connection.py +++ b/proxy/core/connection/connection.py @@ -51,6 +51,7 @@ def connection(self) -> Union[ssl.SSLSocket, socket.socket]: def send(self, data: bytes) -> int: """Users must handle BrokenPipeError exceptions""" # logger.info(data) + logger.info('send to %s, data = |%s|', self.tag, data) return self.connection.send(data) def recv( @@ -65,6 +66,7 @@ def recv( (len(data), self.tag), ) # logger.info(data) + logger.info('recv from %s, data = |%s|', self.tag, data) return memoryview(data) def close(self) -> bool: diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 16cd5096b1..f12fba7a10 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -7,10 +7,6 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. - - .. spelling:: - - reusability """ import logging @@ -64,6 +60,7 @@ class ConnectionPool: def __init__(self) -> None: # Pools of connection per upstream server self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {} + self.handshake_pools: Dict[Tuple[str, int], Dict[int, bytes]] = {} def acquire(self, host: str, port: int) -> Tuple[bool, TcpServerConnection]: """Returns a connection for use with the server.""" @@ -112,4 +109,4 @@ def release(self, conn: TcpServerConnection) -> None: ) assert not conn.is_reusable() # Reset for reusability - conn.reset() + conn.reset() \ No newline at end of file diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index 7aae5371cc..6f699d5938 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -11,7 +11,7 @@ import ssl import socket -from typing import Optional, Union, Tuple +from typing import Optional, Union, Tuple, Dict from ...common.utils import new_socket_connection @@ -27,6 +27,8 @@ def __init__(self, host: str, port: int) -> None: self._conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None self.addr: Tuple[str, int] = (host, int(port)) self.closed = True + self.handshake: Dict(int, bytes) = {} + self.session_id: Optional[bytes] = None @property def connection(self) -> Union[ssl.SSLSocket, socket.socket]: diff --git a/proxy/http/parser/tls.py b/proxy/http/parser/tls.py index 6bee1876d7..6b12eb93e2 100644 --- a/proxy/http/parser/tls.py +++ b/proxy/http/parser/tls.py @@ -8,6 +8,8 @@ :copyright: (c) 2013-present by Abhinav Singh and contributors. :license: BSD, see LICENSE for more details. """ +import os +import binascii import struct import logging @@ -16,6 +18,9 @@ logger = logging.getLogger(__name__) +def pretty_hexlify(raw: bytes) -> str: + hexlified = binascii.hexlify(raw).decode('utf-8') + return ' '.join([hexlified[i: i+2] for i in range(0, len(hexlified), 2)]) TlsContentType = NamedTuple( 'TlsContentType', [ @@ -74,27 +79,110 @@ class TlsClientHello: """TLS Client Hello""" def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> bytes: - self.data = raw - + self.protocol_version: Optional[bytes] = None + self.random: Optional[bytes] = None + self.session_id: Optional[bytes] = None + self.cipher_suite: Optional[bytes] = None + self.compression_method: Optional[bytes] = None + self.extension: Optional[bytes] = None + + def parse(self, raw: bytes) -> (bool, bytes): + try: + idx = 0 + length = len(raw) + self.protocol_version = raw[idx:idx + 2] + idx += 2 + self.random = raw[idx:idx + 32] + idx += 32 + session_length = raw[idx] + self.session_id = raw[idx: idx + 1 + session_length] + idx += 1 + session_length + cipher_suite_length, = struct.unpack('!H', raw[idx: idx + 2]) + self.cipher_suite = raw[idx: idx + 2 + cipher_suite_length] + idx += 2 + cipher_suite_length + compression_method_length = raw[idx] + self.compression_method = raw[idx: idx + 1 + compression_method_length] + idx += 1 + compression_method_length + # extension + if idx == length: + self.extension = [] + else: + extension_length, = struct.unpack('!H', raw[idx: idx + 2]) + self.extension = raw[idx: idx + 2 + extension_length] + idx += 2 + extension_length + return True, raw[idx:] + except Exception as e: + logger.exception(e) + return False, raw + def build(self) -> bytes: - return self.data + # calculate length + return b''.join([bs for bs in [self.protocol_version, self.random, self.session_id, self.cipher_suite, + self.compression_method, self.extension] if bs is not None ]) + + def format(self) -> str: + parts = [] + parts.append('Protocol Version: %s'%(pretty_hexlify(self.protocol_version) if self.protocol_version is not None else '')) + parts.append('Random: %s'%(pretty_hexlify(self.random) if self.random is not None else '')) + parts.append('Session ID: %s'%(pretty_hexlify(self.session_id) if self.session_id is not None else '')) + parts.append('Cipher Suite: %s'%(pretty_hexlify(self.cipher_suite) if self.cipher_suite is not None else '')) + parts.append('Compression Method: %s'%(pretty_hexlify(self.compression_method) if self.compression_method is not None else '')) + parts.append('Extension: %s'%(pretty_hexlify(self.extension) if self.extension is not None else '')) + return os.linesep.join(parts) class TlsServerHello: """TLS Server Hello""" def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> bytes: - self.data = raw + self.protocol_version: Optional[bytes] = None + self.random: Optional[bytes] = None + self.session_id: Optional[bytes] = None + self.cipher_suite: Optional[bytes] = None + self.compression_method: Optional[bytes] = None + self.extension: Optional[bytes] = None + + def parse(self, raw: bytes) -> (bool, bytes): + try: + idx = 0 + length = len(raw) + self.protocol_version = raw[idx:idx + 2] + idx += 2 + self.random = raw[idx:idx + 32] + idx += 32 + session_length = raw[idx] + self.session_id = raw[idx: idx + 1 + session_length] + idx += 1 + session_length + self.cipher_suite = raw[idx: idx + 2] + idx += 2 + compression_method_length = raw[idx] + self.compression_method = raw[idx: idx + 1 + compression_method_length] + idx += 1 + compression_method_length + # extension + if idx == length: + self.extension = [] + else: + extension_length, = struct.unpack('!H', raw[idx: idx + 2]) + self.extension = raw[idx: idx + 2 + extension_length ] + idx += 2 + extension_length + return True, raw[idx:] + except Exception as e: + logger.exception(e) + return False, raw def build(self) -> bytes: - return self.data - + return b''.join([bs for bs in [self.protocol_version, self.random, self.session_id, self.cipher_suite, + self.compression_method, self.extension] if bs is not None ]) + + def format(self) -> str: + parts = [] + parts.append('Protocol Version: %s'%(pretty_hexlify(self.protocol_version) if self.protocol_version is not None else '')) + parts.append('Random: %s'%(pretty_hexlify(self.random) if self.random is not None else '')) + parts.append('Session ID: %s'%(pretty_hexlify(self.session_id) if self.session_id is not None else '')) + parts.append('Cipher Suite: %s'%(pretty_hexlify(self.cipher_suite) if self.cipher_suite is not None else '')) + parts.append('Compression Method: %s'%(pretty_hexlify(self.compression_method) if self.compression_method is not None else '')) + parts.append('Extension: %s'%(pretty_hexlify(self.extension) if self.extension is not None else '')) + return os.linesep.join(parts) class TlsCertificate: """TLS Certificate""" @@ -102,8 +190,9 @@ class TlsCertificate: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: + def parse(self, raw: bytes) -> (bool, bytes): self.data = raw + return True, raw def build(self) -> bytes: return self.data @@ -115,8 +204,9 @@ class TlsServerKeyExchange: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: + def parse(self, raw: bytes) -> (bool, bytes): self.data = raw + return True, raw def build(self) -> bytes: return self.data @@ -128,8 +218,8 @@ class TlsCertificateRequest: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: - self.data = raw + def parse(self, raw: bytes) -> (bool, bytes): + return False, raw def build(self) -> bytes: return self.data @@ -141,8 +231,8 @@ class TlsServerHelloDone: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: - self.data = raw + def parse(self, raw: bytes) -> (bool, bytes): + return False, raw def build(self) -> bytes: return self.data @@ -154,8 +244,8 @@ class TlsCertificateVerify: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: - self.data = raw + def parse(self, raw: bytes) -> (bool, bytes): + return False, raw def build(self) -> bytes: return self.data @@ -167,8 +257,8 @@ class TlsClientKeyExchange: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: - self.data = raw + def parse(self, raw: bytes) -> (bool, bytes): + return False, raw def build(self) -> bytes: return self.data @@ -180,8 +270,8 @@ class TlsFinished: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: - self.data = raw + def parse(self, raw: bytes) -> (bool, bytes): + return False, raw def build(self) -> bytes: return self.data @@ -191,10 +281,10 @@ class TlsHandshake: """TLS Handshake""" def __init__(self) -> None: - self.handshake_type: int = tlsHandshakeType.OTHER - self.length: int = 0 + self.msg_type: int = tlsHandshakeType.OTHER + self.length: Optional[bytes] = None self.hello_request: Optional[TlsHelloRequest] = None - self.client_hellp: Optional[TlsClientHello] = None + self.client_hello: Optional[TlsClientHello] = None self.server_hello: Optional[TlsServerHello] = None self.certificate: Optional[TlsCertificate] = None self.server_key_exchange: Optional[TlsServerKeyExchange] = None @@ -205,66 +295,82 @@ def __init__(self) -> None: self.finished: Optional[TlsFinished] = None self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: + def parse(self, raw: bytes) -> (bool, bytes): length = len(raw) if length < 4: logger.debug('invalid data, len(raw) = %s', legth) return raw else: - payload_length, = struct.unpack('!H', b'\x00' + raw[1:4]) + payload_length, = struct.unpack('!I', b'\x00' + raw[1:4]) self.length = payload_length if length < 4 + payload_length: logger.debug('incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length) - return raw + return False, raw else: # parse - self.handshake_type = raw[0] + self.msg_type = raw[0] + self.length = raw[1:4] self.data = raw[: 4 + payload_length] payload = raw[4: 4 + payload_length] - if self.handshake_type == tlsHandshakeType.HELLO_REQUEST: + if self.msg_type == tlsHandshakeType.HELLO_REQUEST: # parse hello request self.hello_request = TlsHelloRequest() self.hello_request.parse(payload) - elif self.handshake_type == tlsHandshakeType.CLIENT_HELLO: + elif self.msg_type == tlsHandshakeType.CLIENT_HELLO: # parse client hello - self.client_hellp = TlsClientHello() - self.client_hellp.parse(payload) - elif self.handshake_type == tlsHandshakeType.SERVER_HELLO: + self.client_hello = TlsClientHello() + self.client_hello.parse(payload) + elif self.msg_type == tlsHandshakeType.SERVER_HELLO: # parse server hello self.server_hello = TlsServerHello() self.server_hello.parse(payload) - elif self.handshake_type == tlsHandshakeType.CERTIFICATE: + elif self.msg_type == tlsHandshakeType.CERTIFICATE: # parse certficate self.certificate = TlsCertificate() self.certificate.parse(payload) - elif self.handshake_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: + elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: # parse server key exchange self.server_key_exchange = TlsServerKeyExchange() self.server_key_exchange.parse(payload) - elif self.handshake_type == tlsHandshakeType.CERTIFICATE_REQUEST: + elif self.msg_type == tlsHandshakeType.CERTIFICATE_REQUEST: # parse certificate request self.certificate_request = TlsCertificateRequest() self.certificate_request.parse(payload) - elif self.handshake_type == tlsHandshakeType.SERVER_HELLO_DONE: + elif self.msg_type == tlsHandshakeType.SERVER_HELLO_DONE: # parse server hello done self.server_hello_done = TlsServerHelloDone() self.server_hello_done.parse(payload) - elif self.handshake_type == tlsHandshakeType.CERTIFICATE_VERIFY: + elif self.msg_type == tlsHandshakeType.CERTIFICATE_VERIFY: # parse certificate verify self.certificate_verify = TlsCertificateVerify() - self.certificate_verify.parse(payload) - elif self.handshake_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE: + rself.certificate_verify.parse(payload) + elif self.msg_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE: # parse client key exchange self.client_key_exchange = TlsClientKeyExchange() self.client_key_exchange.parse(payload) - elif self.handshake_type == tlsHandshakeType.FINISHED: + elif self.msg_type == tlsHandshakeType.FINISHED: # parse finished self.finished = TlsFinished() self.finished.parse(payload) - return raw[4 + payload_length:] + return True, raw[4 + payload_length:] def build(self) -> bytes: - return self.data + data = bytes() + data += bytes([self.msg_type]) + payload = bytes() + if self.msg_type == tlsHandshakeType.CLIENT_HELLO: + payload = self.client_hello.build() + elif self.msg_type == tlsHandshakeType.SERVER_HELLO: + payload = self.server_hello.build() + elif self.msg_type == tlsHandshakeType.CERTIFICATE: + payload = self.certificate.build() + elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: + payload = self.server_key_exchange.build() + # calculate length + length = struct.pack('!I', len(payload))[1:] + data += length + data += payload + return data class TlsParser: @@ -273,38 +379,49 @@ class TlsParser: def __init__(self) -> None: self.content_type: int = tlsContentType.OTHER self.protocol_version: Optional[TlsProtocolVersion] = None - self.length: int = 0 - self.data: Optional[bytes] = None + self.length: Optional[bytes] = None # only parse hand shake payload temporary self.handshake: Optional[TlsHandshake] = None + self.certificate: Optional[TlsCertificate] - def parse(self, raw: bytes) -> bytes: + def parse(self, raw: bytes) -> (bool, bytes): """parse TLS fragmentation Ref: https://datatracker.ietf.org/doc/html/rfc5246#page-15 + https://datatracker.ietf.org/doc/html/rfc5077#page-3 + https://datatracker.ietf.org/doc/html/rfc8446#page-10 """ length = len(raw) if length < 5: logger.debug('invalid data, len(raw) = %s', length) - return raw + return False, raw else: payload_length, = struct.unpack('!H', raw[3:5]) - self.length = payload_length + self.protocol_version if length < 5 + payload_length: logger.debug('incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length) - return raw + return False, raw else: # parse self.content_type = raw[0] - self.data = raw[: 5 + payload_length] + self.protocol_version = raw[1:3] + self.length = raw[3:5] payload = raw[5:5 + payload_length] if self.content_type == tlsContentType.HANDSHAKE: # parse handshake self.handshake = TlsHandshake() self.handshake.parse(payload) - self.data = raw[:5 + payload_length] - return raw[5 + payload_length:] + return True, raw[5 + payload_length:] def build(self) -> bytes: - return self.data + data = bytes() + data += bytes([self.content_type]) + data += self.protocol_version + payload = bytes() + if self.content_type == tlsContentType.HANDSHAKE: + payload += self.handshake.build() + length = struct.pack('!H', len(payload)) + data += length + data += payload + return data diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 50cee611d9..5b4a01f1fe 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -31,6 +31,7 @@ from ..plugin import HttpProtocolHandlerPlugin from ..exception import HttpProtocolException, ProxyConnectionFailed from ..parser import HttpParser, httpParserStates, httpParserTypes +from ..parser.tls import TlsParser, TlsHandshake, tlsContentType, tlsHandshakeType from ...common.types import Readables, Writables from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE @@ -268,7 +269,7 @@ def read_from_descriptors(self, r: Readables) -> bool: and self.upstream \ and not self.upstream.closed \ and self.upstream.connection in r: - logger.debug('Server is ready for reads, reading...') + # logger.debug('Server is ready for reads, reading...') try: raw = self.upstream.recv(self.flags.server_recvbuf_size) except TimeoutError as e: @@ -308,7 +309,31 @@ def read_from_descriptors(self, r: Readables) -> bool: return self._close_and_release() else: return False - + # parse server response + tls = TlsParser() + flag, left = tls.parse(raw.tobytes()) + if tls.content_type == tlsContentType.HANDSHAKE: + # save + logger.debug('receive TLS handshake, handshake type = %s', tls.handshake.msg_type) + import binascii + open('%s_%s_%s.data'%(text_(self.request.host), self.request.port, tls.handshake.msg_type), 'w+').write(binascii.hexlify(raw).decode('utf-8')) + if tls.handshake.msg_type == tlsHandshakeType.SERVER_HELLO: + logger.debug('got TLS SERVER_HELLO response, session_id = |%s|', binascii.hexlify(tls.handshake.server_hello.session_id)) + logger.debug('server hello : %s', tls.handshake.server_hello.format()) + self.upstream.session_id = tls.handshake.server_hello.session_id + # self.upstream.handshake[tlsHandshakeType.SERVER_HELLO] = raw.tobytes() + self.upstream.handshake[tlsHandshakeType.SERVER_HELLO] = tls.build() + # parse remain + flag, left = tls.parse(left) + logger.debug('flag = %s, msg_type = %s', flag, tls.handshake.msg_type) + if tls.handshake.msg_type == tlsHandshakeType.CERTIFICATE: + self.upstream.handshake[tlsHandshakeType.CERTIFICATE] = tls.build() + # parse remain + if len(left): + flag, left = tls.parse(left) + logger.debug('flag = %s, msg_type = %s', flag, tls.handshake.msg_type) + if tls.handshake.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: + self.upstream.handshake[tlsHandshakeType.SERVER_KEY_EXCHANGE] = tls.build() for plugin in self.plugins.values(): raw = plugin.handle_upstream_chunk(raw) @@ -516,6 +541,34 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: # For scenarios where we cannot peek into the data, # simply queue for upstream server. else: + if self.request.is_https_tunnel(): + # parse raw + # parse server response + tls = TlsParser() + tls.parse(raw) + if tls.content_type == tlsContentType.HANDSHAKE: + # save + import binascii + logger.debug('receive client tls handshake, handshake type = %s', tls.handshake.msg_type) + if tls.handshake.msg_type == tlsHandshakeType.CLIENT_HELLO: + logger.debug('client hello : %s', tls.handshake.client_hello.format()) + # connection reused + if tlsHandshakeType.SERVER_HELLO in self.upstream.handshake: + # send cached SERVER HELLO response to client + logger.debug('return cached SERVER HELLO to client') + data = self.upstream.handshake[tlsHandshakeType.SERVER_HELLO] + self.upstream.handshake[tlsHandshakeType.CERTIFICATE] + \ + self.upstream.handshake[tlsHandshakeType.SERVER_KEY_EXCHANGE] + self.client.queue(memoryview(data)) + return None + # check if close_notify + bs = raw.tobytes() + logger.debug(bs) + logger.debug('bs[0] = |%s|', bs[0]) + if bs[0] == 0x15: + logger.debug('close notify, just ignore') + return None + else: + logger.debug('not close notify') self.upstream.queue(raw) return None return raw diff --git a/tests/http/test_tls_parser.py b/tests/http/test_tls_parser.py index 8fd7b43380..5fc51d4be5 100644 --- a/tests/http/test_tls_parser.py +++ b/tests/http/test_tls_parser.py @@ -11,6 +11,7 @@ import unittest import binascii import re +import os from proxy.http.parser.tls import TlsParser, TlsHandshake from proxy.http.parser.tls import tlsContentType, tlsHandshakeType @@ -37,8 +38,21 @@ def test_parse_client_hello(self) -> None: tls = TlsParser() tls.parse(self.unhexlify(data)) self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE) - self.assertEqual(tls.length, 0xa5) - self.assertEqual(tls.handshake.handshake_type, tlsHandshakeType.CLIENT_HELLO) - self.assertEqual(tls.handshake.length, 0xa1) + self.assertEqual(tls.length, b'\x00\xa5') + self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.CLIENT_HELLO) + self.assertEqual(tls.handshake.length, b'\x00\x00\xa1') self.assertEqual(len(tls.handshake.build()), 0xa1 + 0x04) + self.assertEqual(tls.handshake.client_hello.protocol_version, b'\x03\x03') + self.assertEqual(tls.handshake.client_hello.random, self.unhexlify('00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f')) + + def test_parse_server_hello(self) -> None: + data = open(os.path.join(os.path.dirname(__file__), 'tls_server_hello.data'), 'r').read() + tls = TlsParser() + tls.parse(self.unhexlify(data)) + self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE) + self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.SERVER_HELLO) + self.assertEqual(tls.handshake.server_hello.protocol_version, b'\x03\x03') + print(tls.handshake.server_hello.format()) + + From 15da1148ac582ddc8ee0ab303c670b552ecbe1da Mon Sep 17 00:00:00 2001 From: Jerry Date: Fri, 10 Dec 2021 17:37:39 +0800 Subject: [PATCH 04/14] fix bugs when using connection pool with https fix bugs when using connection pool with https --- proxy/http/proxy/server.py | 62 +++++--------------------------------- 1 file changed, 7 insertions(+), 55 deletions(-) diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index a8b43e34ee..198fb8c797 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -31,7 +31,7 @@ from ..plugin import HttpProtocolHandlerPlugin from ..exception import HttpProtocolException, ProxyConnectionFailed from ..parser import HttpParser, httpParserStates, httpParserTypes -from ..parser.tls import TlsParser, TlsHandshake, tlsContentType, tlsHandshakeType +# from ..parser.tls import TlsParser, TlsHandshake, tlsContentType, tlsHandshakeType from ...common.types import Readables, Writables from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE @@ -304,31 +304,6 @@ async def read_from_descriptors(self, r: Readables) -> bool: return self._close_and_release() else: return False - # parse server response - tls = TlsParser() - flag, left = tls.parse(raw.tobytes()) - if tls.content_type == tlsContentType.HANDSHAKE: - # save - logger.debug('receive TLS handshake, handshake type = %s', tls.handshake.msg_type) - import binascii - open('%s_%s_%s.data'%(text_(self.request.host), self.request.port, tls.handshake.msg_type), 'w+').write(binascii.hexlify(raw).decode('utf-8')) - if tls.handshake.msg_type == tlsHandshakeType.SERVER_HELLO: - logger.debug('got TLS SERVER_HELLO response, session_id = |%s|', binascii.hexlify(tls.handshake.server_hello.session_id)) - logger.debug('server hello : %s', tls.handshake.server_hello.format()) - self.upstream.session_id = tls.handshake.server_hello.session_id - # self.upstream.handshake[tlsHandshakeType.SERVER_HELLO] = raw.tobytes() - self.upstream.handshake[tlsHandshakeType.SERVER_HELLO] = tls.build() - # parse remain - flag, left = tls.parse(left) - logger.debug('flag = %s, msg_type = %s', flag, tls.handshake.msg_type) - if tls.handshake.msg_type == tlsHandshakeType.CERTIFICATE: - self.upstream.handshake[tlsHandshakeType.CERTIFICATE] = tls.build() - # parse remain - if len(left): - flag, left = tls.parse(left) - logger.debug('flag = %s, msg_type = %s', flag, tls.handshake.msg_type) - if tls.handshake.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: - self.upstream.handshake[tlsHandshakeType.SERVER_KEY_EXCHANGE] = tls.build() for plugin in self.plugins.values(): raw = plugin.handle_upstream_chunk(raw) @@ -536,34 +511,6 @@ def on_client_data(self, raw: memoryview) -> Optional[memoryview]: # For scenarios where we cannot peek into the data, # simply queue for upstream server. else: - if self.request.is_https_tunnel(): - # parse raw - # parse server response - tls = TlsParser() - tls.parse(raw) - if tls.content_type == tlsContentType.HANDSHAKE: - # save - import binascii - logger.debug('receive client tls handshake, handshake type = %s', tls.handshake.msg_type) - if tls.handshake.msg_type == tlsHandshakeType.CLIENT_HELLO: - logger.debug('client hello : %s', tls.handshake.client_hello.format()) - # connection reused - if tlsHandshakeType.SERVER_HELLO in self.upstream.handshake: - # send cached SERVER HELLO response to client - logger.debug('return cached SERVER HELLO to client') - data = self.upstream.handshake[tlsHandshakeType.SERVER_HELLO] + self.upstream.handshake[tlsHandshakeType.CERTIFICATE] + \ - self.upstream.handshake[tlsHandshakeType.SERVER_KEY_EXCHANGE] - self.client.queue(memoryview(data)) - return None - # check if close_notify - bs = raw.tobytes() - logger.debug(bs) - logger.debug('bs[0] = |%s|', bs[0]) - if bs[0] == 0x15: - logger.debug('close notify, just ignore') - return None - else: - logger.debug('not close notify') self.upstream.queue(raw) return None return raw @@ -847,7 +794,12 @@ def wrap_server(self) -> bool: assert isinstance(self.upstream.connection, socket.socket) do_close = False try: - self.upstream.wrap(text_(self.request.host), self.flags.ca_file) + # check if has wrapped already + logger.debug('type(self.upstream) = %s', type(self.upstream)) + if not isinstance(self.upstream.connection, ssl.SSLSocket): + self.upstream.wrap(text_(self.request.host), self.flags.ca_file) + else: + logger.debug('self.upstream is ssl.SSLSocket already, do not need to wrap') except ssl.SSLCertVerificationError: # Server raised certificate verification error # When --disable-interception-on-ssl-cert-verification-error flag is on, # we will cache such upstream hosts and avoid intercepting them for future From ed21988ca9ab2d8e8d504779f03e54440b702efe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Dec 2021 09:11:42 +0000 Subject: [PATCH 05/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- proxy/core/connection/pool.py | 2 +- proxy/http/parser/tls.py | 24 ++++++++++++++---------- tests/http/test_tls_parser.py | 3 --- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index f12fba7a10..43ba407274 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -109,4 +109,4 @@ def release(self, conn: TcpServerConnection) -> None: ) assert not conn.is_reusable() # Reset for reusability - conn.reset() \ No newline at end of file + conn.reset() diff --git a/proxy/http/parser/tls.py b/proxy/http/parser/tls.py index 6b12eb93e2..ac6408ad95 100644 --- a/proxy/http/parser/tls.py +++ b/proxy/http/parser/tls.py @@ -30,7 +30,7 @@ def pretty_hexlify(raw: bytes) -> str: ('APPLICATION_DATA', int), ('OTHER', int), ], - ) +) tlsContentType = TlsContentType(20, 21, 22, 23, 255) class TlsProtocolVersion: @@ -59,7 +59,7 @@ def set_value(major: int, minor: int) -> None: ('FINISHED', int), ('OTHER', int), ], - ) +) tlsHandshakeType = TlsHandshakeType(0, 1, 2, 11, 12, 13, 14, 15, 16, 20, 255) class TlsHelloRequest: @@ -114,11 +114,14 @@ def parse(self, raw: bytes) -> (bool, bytes): except Exception as e: logger.exception(e) return False, raw - + def build(self) -> bytes: # calculate length - return b''.join([bs for bs in [self.protocol_version, self.random, self.session_id, self.cipher_suite, - self.compression_method, self.extension] if bs is not None ]) + return b''.join([ + bs for bs in [ + self.protocol_version, self.random, self.session_id, self.cipher_suite, + self.compression_method, self.extension, + ] if bs is not None ]) def format(self) -> str: parts = [] @@ -163,7 +166,7 @@ def parse(self, raw: bytes) -> (bool, bytes): self.extension = [] else: extension_length, = struct.unpack('!H', raw[idx: idx + 2]) - self.extension = raw[idx: idx + 2 + extension_length ] + self.extension = raw[idx: idx + 2 + extension_length] idx += 2 + extension_length return True, raw[idx:] except Exception as e: @@ -171,8 +174,11 @@ def parse(self, raw: bytes) -> (bool, bytes): return False, raw def build(self) -> bytes: - return b''.join([bs for bs in [self.protocol_version, self.random, self.session_id, self.cipher_suite, - self.compression_method, self.extension] if bs is not None ]) + return b''.join([ + bs for bs in [ + self.protocol_version, self.random, self.session_id, self.cipher_suite, + self.compression_method, self.extension, + ] if bs is not None ]) def format(self) -> str: parts = [] @@ -423,5 +429,3 @@ def build(self) -> bytes: data += length data += payload return data - - diff --git a/tests/http/test_tls_parser.py b/tests/http/test_tls_parser.py index 5fc51d4be5..d49763d3c9 100644 --- a/tests/http/test_tls_parser.py +++ b/tests/http/test_tls_parser.py @@ -53,6 +53,3 @@ def test_parse_server_hello(self) -> None: self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.SERVER_HELLO) self.assertEqual(tls.handshake.server_hello.protocol_version, b'\x03\x03') print(tls.handshake.server_hello.format()) - - - From a44b67c1e7384280f4fbc3f5ff949efefaf33757 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Fri, 17 Dec 2021 18:32:59 +0530 Subject: [PATCH 06/14] Fix syntax, lint, spellcheck errors --- README.md | 2 +- proxy/core/connection/server.py | 2 +- proxy/http/parser/tls.py | 180 +++++++++++++++++++++++++------- tests/http/test_tls_parser.py | 34 ++++-- 4 files changed, 167 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 59efaf4285..d358d236a5 100644 --- a/README.md +++ b/README.md @@ -2073,7 +2073,7 @@ usage: -m [-h] [--enable-events] [--enable-conn-pool] [--threadless] [--filtered-url-regex-config FILTERED_URL_REGEX_CONFIG] [--cloudflare-dns-mode CLOUDFLARE_DNS_MODE] -proxy.py v2.4.0rc2.dev21+g20b3eb1.d20211203 +proxy.py v2.4.0rc3.dev33+gc341594.d20211214 options: -h, --help show this help message and exit diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index 6f699d5938..7ab09dc904 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -27,7 +27,7 @@ def __init__(self, host: str, port: int) -> None: self._conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None self.addr: Tuple[str, int] = (host, int(port)) self.closed = True - self.handshake: Dict(int, bytes) = {} + self.handshake: Dict[int, bytes] = {} self.session_id: Optional[bytes] = None @property diff --git a/proxy/http/parser/tls.py b/proxy/http/parser/tls.py index ac6408ad95..c3b41c69d7 100644 --- a/proxy/http/parser/tls.py +++ b/proxy/http/parser/tls.py @@ -13,15 +13,17 @@ import struct import logging -from typing import NamedTuple, Optional +from typing import NamedTuple, Optional, Tuple logger = logging.getLogger(__name__) + def pretty_hexlify(raw: bytes) -> str: hexlified = binascii.hexlify(raw).decode('utf-8') return ' '.join([hexlified[i: i+2] for i in range(0, len(hexlified), 2)]) + TlsContentType = NamedTuple( 'TlsContentType', [ ('CHANGE_CIPHER_SPEC', int), @@ -33,6 +35,7 @@ def pretty_hexlify(raw: bytes) -> str: ) tlsContentType = TlsContentType(20, 21, 22, 23, 255) + class TlsProtocolVersion: """Protocol Version""" @@ -40,7 +43,7 @@ def __init__(self) -> None: self.major = 0 self.minor = 0 - def set_value(major: int, minor: int) -> None: + def set_value(self, major: int, minor: int) -> None: self.major = major self.minor = minor @@ -62,16 +65,18 @@ def set_value(major: int, minor: int) -> None: ) tlsHandshakeType = TlsHandshakeType(0, 1, 2, 11, 12, 13, 14, 15, 16, 20, 255) + class TlsHelloRequest: """TLS Hello Request""" def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> bytes: + def parse(self, raw: bytes) -> None: self.data = raw def build(self) -> bytes: + assert self.data return self.data @@ -86,7 +91,7 @@ def __init__(self) -> None: self.compression_method: Optional[bytes] = None self.extension: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: try: idx = 0 length = len(raw) @@ -101,11 +106,14 @@ def parse(self, raw: bytes) -> (bool, bytes): self.cipher_suite = raw[idx: idx + 2 + cipher_suite_length] idx += 2 + cipher_suite_length compression_method_length = raw[idx] - self.compression_method = raw[idx: idx + 1 + compression_method_length] + self.compression_method = raw[ + idx: idx + + 1 + compression_method_length + ] idx += 1 + compression_method_length # extension if idx == length: - self.extension = [] + self.extension = b'' else: extension_length, = struct.unpack('!H', raw[idx: idx + 2]) self.extension = raw[idx: idx + 2 + extension_length] @@ -121,16 +129,52 @@ def build(self) -> bytes: bs for bs in [ self.protocol_version, self.random, self.session_id, self.cipher_suite, self.compression_method, self.extension, - ] if bs is not None ]) + ] if bs is not None + ]) def format(self) -> str: parts = [] - parts.append('Protocol Version: %s'%(pretty_hexlify(self.protocol_version) if self.protocol_version is not None else '')) - parts.append('Random: %s'%(pretty_hexlify(self.random) if self.random is not None else '')) - parts.append('Session ID: %s'%(pretty_hexlify(self.session_id) if self.session_id is not None else '')) - parts.append('Cipher Suite: %s'%(pretty_hexlify(self.cipher_suite) if self.cipher_suite is not None else '')) - parts.append('Compression Method: %s'%(pretty_hexlify(self.compression_method) if self.compression_method is not None else '')) - parts.append('Extension: %s'%(pretty_hexlify(self.extension) if self.extension is not None else '')) + parts.append( + 'Protocol Version: %s' % ( + pretty_hexlify( + self.protocol_version, + ) if self.protocol_version is not None else '' + ), + ) + parts.append( + 'Random: %s' % ( + pretty_hexlify(self.random) + if self.random is not None else '' + ), + ) + parts.append( + 'Session ID: %s' % ( + pretty_hexlify( + self.session_id, + ) if self.session_id is not None else '' + ), + ) + parts.append( + 'Cipher Suite: %s' % ( + pretty_hexlify( + self.cipher_suite, + ) if self.cipher_suite is not None else '' + ), + ) + parts.append( + 'Compression Method: %s' % ( + pretty_hexlify( + self.compression_method, + ) if self.compression_method is not None else '' + ), + ) + parts.append( + 'Extension: %s' % ( + pretty_hexlify( + self.extension, + ) if self.extension is not None else '' + ), + ) return os.linesep.join(parts) @@ -145,7 +189,7 @@ def __init__(self) -> None: self.compression_method: Optional[bytes] = None self.extension: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: try: idx = 0 length = len(raw) @@ -159,11 +203,14 @@ def parse(self, raw: bytes) -> (bool, bytes): self.cipher_suite = raw[idx: idx + 2] idx += 2 compression_method_length = raw[idx] - self.compression_method = raw[idx: idx + 1 + compression_method_length] + self.compression_method = raw[ + idx: idx + + 1 + compression_method_length + ] idx += 1 + compression_method_length # extension if idx == length: - self.extension = [] + self.extension = b'' else: extension_length, = struct.unpack('!H', raw[idx: idx + 2]) self.extension = raw[idx: idx + 2 + extension_length] @@ -178,29 +225,67 @@ def build(self) -> bytes: bs for bs in [ self.protocol_version, self.random, self.session_id, self.cipher_suite, self.compression_method, self.extension, - ] if bs is not None ]) + ] if bs is not None + ]) def format(self) -> str: parts = [] - parts.append('Protocol Version: %s'%(pretty_hexlify(self.protocol_version) if self.protocol_version is not None else '')) - parts.append('Random: %s'%(pretty_hexlify(self.random) if self.random is not None else '')) - parts.append('Session ID: %s'%(pretty_hexlify(self.session_id) if self.session_id is not None else '')) - parts.append('Cipher Suite: %s'%(pretty_hexlify(self.cipher_suite) if self.cipher_suite is not None else '')) - parts.append('Compression Method: %s'%(pretty_hexlify(self.compression_method) if self.compression_method is not None else '')) - parts.append('Extension: %s'%(pretty_hexlify(self.extension) if self.extension is not None else '')) + parts.append( + 'Protocol Version: %s' % ( + pretty_hexlify( + self.protocol_version, + ) if self.protocol_version is not None else '' + ), + ) + parts.append( + 'Random: %s' % ( + pretty_hexlify(self.random) + if self.random is not None else '' + ), + ) + parts.append( + 'Session ID: %s' % ( + pretty_hexlify( + self.session_id, + ) if self.session_id is not None else '' + ), + ) + parts.append( + 'Cipher Suite: %s' % ( + pretty_hexlify( + self.cipher_suite, + ) if self.cipher_suite is not None else '' + ), + ) + parts.append( + 'Compression Method: %s' % ( + pretty_hexlify( + self.compression_method, + ) if self.compression_method is not None else '' + ), + ) + parts.append( + 'Extension: %s' % ( + pretty_hexlify( + self.extension, + ) if self.extension is not None else '' + ), + ) return os.linesep.join(parts) + class TlsCertificate: """TLS Certificate""" def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: self.data = raw return True, raw def build(self) -> bytes: + assert self.data return self.data @@ -210,11 +295,12 @@ class TlsServerKeyExchange: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: self.data = raw return True, raw def build(self) -> bytes: + assert self.data return self.data @@ -224,10 +310,11 @@ class TlsCertificateRequest: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: return False, raw def build(self) -> bytes: + assert self.data return self.data @@ -237,23 +324,25 @@ class TlsServerHelloDone: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: return False, raw def build(self) -> bytes: + assert self.data return self.data class TlsCertificateVerify: - """TLS Certificate Vefiry""" + """TLS Certificate Verify""" def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: return False, raw def build(self) -> bytes: + assert self.data return self.data @@ -263,10 +352,11 @@ class TlsClientKeyExchange: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: return False, raw def build(self) -> bytes: + assert self.data return self.data @@ -276,10 +366,11 @@ class TlsFinished: def __init__(self) -> None: self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: return False, raw def build(self) -> bytes: + assert self.data return self.data @@ -301,16 +392,18 @@ def __init__(self) -> None: self.finished: Optional[TlsFinished] = None self.data: Optional[bytes] = None - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: length = len(raw) if length < 4: - logger.debug('invalid data, len(raw) = %s', legth) - return raw + logger.debug('invalid data, len(raw) = %s', length) + return False, raw else: payload_length, = struct.unpack('!I', b'\x00' + raw[1:4]) self.length = payload_length if length < 4 + payload_length: - logger.debug('incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length) + logger.debug( + 'incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length, + ) return False, raw else: # parse @@ -331,7 +424,7 @@ def parse(self, raw: bytes) -> (bool, bytes): self.server_hello = TlsServerHello() self.server_hello.parse(payload) elif self.msg_type == tlsHandshakeType.CERTIFICATE: - # parse certficate + # parse certificate self.certificate = TlsCertificate() self.certificate.parse(payload) elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: @@ -349,7 +442,7 @@ def parse(self, raw: bytes) -> (bool, bytes): elif self.msg_type == tlsHandshakeType.CERTIFICATE_VERIFY: # parse certificate verify self.certificate_verify = TlsCertificateVerify() - rself.certificate_verify.parse(payload) + self.certificate_verify.parse(payload) elif self.msg_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE: # parse client key exchange self.client_key_exchange = TlsClientKeyExchange() @@ -365,12 +458,16 @@ def build(self) -> bytes: data += bytes([self.msg_type]) payload = bytes() if self.msg_type == tlsHandshakeType.CLIENT_HELLO: + assert self.client_hello payload = self.client_hello.build() elif self.msg_type == tlsHandshakeType.SERVER_HELLO: + assert self.server_hello payload = self.server_hello.build() elif self.msg_type == tlsHandshakeType.CERTIFICATE: + assert self.certificate payload = self.certificate.build() elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: + assert self.server_key_exchange payload = self.server_key_exchange.build() # calculate length length = struct.pack('!I', len(payload))[1:] @@ -390,7 +487,7 @@ def __init__(self) -> None: self.handshake: Optional[TlsHandshake] = None self.certificate: Optional[TlsCertificate] - def parse(self, raw: bytes) -> (bool, bytes): + def parse(self, raw: bytes) -> Tuple[bool, bytes]: """parse TLS fragmentation Ref: https://datatracker.ietf.org/doc/html/rfc5246#page-15 https://datatracker.ietf.org/doc/html/rfc5077#page-3 @@ -404,11 +501,14 @@ def parse(self, raw: bytes) -> (bool, bytes): payload_length, = struct.unpack('!H', raw[3:5]) self.protocol_version if length < 5 + payload_length: - logger.debug('incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length) + logger.debug( + 'incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length, + ) return False, raw else: # parse self.content_type = raw[0] + # ??? self.protocol_version = raw[1:3] self.length = raw[3:5] payload = raw[5:5 + payload_length] @@ -421,9 +521,11 @@ def parse(self, raw: bytes) -> (bool, bytes): def build(self) -> bytes: data = bytes() data += bytes([self.content_type]) + # ??? data += self.protocol_version payload = bytes() if self.content_type == tlsContentType.HANDSHAKE: + assert self.handshake payload += self.handshake.build() length = struct.pack('!H', len(payload)) data += length diff --git a/tests/http/test_tls_parser.py b/tests/http/test_tls_parser.py index d49763d3c9..c6b99ab35f 100644 --- a/tests/http/test_tls_parser.py +++ b/tests/http/test_tls_parser.py @@ -13,20 +13,18 @@ import re import os -from proxy.http.parser.tls import TlsParser, TlsHandshake +from proxy.http.parser.tls import TlsParser from proxy.http.parser.tls import tlsContentType, tlsHandshakeType class TestTlsParser(unittest.TestCase): - """ - Ref: https://tls.ulfheim.net/ - """ + """Ref: https://tls.ulfheim.net/""" def unhexlify(self, raw: str) -> bytes: return binascii.unhexlify(re.sub(r'\s', '', raw)) def test_parse_client_hello(self) -> None: - data =""" + data = """ 16 03 01 00 a5 01 00 00 a1 03 03 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 00 00 20 cc a8 cc a9 c0 2f c0 30 c0 2b c0 2c c0 13 c0 09 c0 14 c0 0a 00 9c 00 9d 00 2f 00 35 c0 12 00 0a 01 00 00 58 00 00 00 18 00 16 00 00 13 65 78 61 6d 70 @@ -34,22 +32,38 @@ def test_parse_client_hello(self) -> None: 17 00 18 00 19 00 0b 00 02 01 00 00 0d 00 12 00 10 04 01 04 03 05 01 05 03 06 01 06 03 02 01 02 03 ff 01 00 01 00 00 12 00 00 """ - raw = b'\x81\x85\xc6\ti\x8d\xael\x05\xe1\xa9' tls = TlsParser() tls.parse(self.unhexlify(data)) self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE) self.assertEqual(tls.length, b'\x00\xa5') + assert tls.handshake self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.CLIENT_HELLO) self.assertEqual(tls.handshake.length, b'\x00\x00\xa1') self.assertEqual(len(tls.handshake.build()), 0xa1 + 0x04) - self.assertEqual(tls.handshake.client_hello.protocol_version, b'\x03\x03') - self.assertEqual(tls.handshake.client_hello.random, self.unhexlify('00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f')) + assert tls.handshake.client_hello + self.assertEqual( + tls.handshake.client_hello.protocol_version, b'\x03\x03', + ) + self.assertEqual( + tls.handshake.client_hello.random, self.unhexlify( + '00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f', + ), + ) def test_parse_server_hello(self) -> None: - data = open(os.path.join(os.path.dirname(__file__), 'tls_server_hello.data'), 'r').read() + data = open( + os.path.join( + os.path.dirname(__file__), + 'tls_server_hello.data', + ), 'r', + ).read() tls = TlsParser() tls.parse(self.unhexlify(data)) self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE) + assert tls.handshake self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.SERVER_HELLO) - self.assertEqual(tls.handshake.server_hello.protocol_version, b'\x03\x03') + assert tls.handshake.server_hello + self.assertEqual( + tls.handshake.server_hello.protocol_version, b'\x03\x03', + ) print(tls.handshake.server_hello.format()) From d6e90a81101803bb39fd6bde9b99137214ffc492 Mon Sep 17 00:00:00 2001 From: Abhinav Singh Date: Fri, 17 Dec 2021 18:45:02 +0530 Subject: [PATCH 07/14] Fix flake8 errors --- proxy/http/parser/tls.py | 172 +++++++++++++++++----------------- proxy/http/proxy/server.py | 3 +- tests/http/test_tls_parser.py | 2 +- 3 files changed, 88 insertions(+), 89 deletions(-) diff --git a/proxy/http/parser/tls.py b/proxy/http/parser/tls.py index c3b41c69d7..6b017e9cac 100644 --- a/proxy/http/parser/tls.py +++ b/proxy/http/parser/tls.py @@ -126,19 +126,19 @@ def parse(self, raw: bytes) -> Tuple[bool, bytes]: def build(self) -> bytes: # calculate length return b''.join([ - bs for bs in [ + bs for bs in ( self.protocol_version, self.random, self.session_id, self.cipher_suite, self.compression_method, self.extension, - ] if bs is not None + ) if bs is not None ]) def format(self) -> str: parts = [] parts.append( 'Protocol Version: %s' % ( - pretty_hexlify( - self.protocol_version, - ) if self.protocol_version is not None else '' + pretty_hexlify(self.protocol_version) + if self.protocol_version is not None + else '' ), ) parts.append( @@ -149,30 +149,30 @@ def format(self) -> str: ) parts.append( 'Session ID: %s' % ( - pretty_hexlify( - self.session_id, - ) if self.session_id is not None else '' + pretty_hexlify(self.session_id) + if self.session_id is not None + else '' ), ) parts.append( 'Cipher Suite: %s' % ( - pretty_hexlify( - self.cipher_suite, - ) if self.cipher_suite is not None else '' + pretty_hexlify(self.cipher_suite) + if self.cipher_suite is not None + else '' ), ) parts.append( 'Compression Method: %s' % ( - pretty_hexlify( - self.compression_method, - ) if self.compression_method is not None else '' + pretty_hexlify(self.compression_method) + if self.compression_method is not None + else '' ), ) parts.append( 'Extension: %s' % ( - pretty_hexlify( - self.extension, - ) if self.extension is not None else '' + pretty_hexlify(self.extension) + if self.extension is not None + else '' ), ) return os.linesep.join(parts) @@ -222,53 +222,54 @@ def parse(self, raw: bytes) -> Tuple[bool, bytes]: def build(self) -> bytes: return b''.join([ - bs for bs in [ + bs for bs in ( self.protocol_version, self.random, self.session_id, self.cipher_suite, self.compression_method, self.extension, - ] if bs is not None + ) if bs is not None ]) def format(self) -> str: parts = [] parts.append( 'Protocol Version: %s' % ( - pretty_hexlify( - self.protocol_version, - ) if self.protocol_version is not None else '' + pretty_hexlify(self.protocol_version) + if self.protocol_version is not None + else '' ), ) parts.append( 'Random: %s' % ( pretty_hexlify(self.random) - if self.random is not None else '' + if self.random is not None + else '' ), ) parts.append( 'Session ID: %s' % ( - pretty_hexlify( - self.session_id, - ) if self.session_id is not None else '' + pretty_hexlify(self.session_id) + if self.session_id is not None + else '' ), ) parts.append( 'Cipher Suite: %s' % ( - pretty_hexlify( - self.cipher_suite, - ) if self.cipher_suite is not None else '' + pretty_hexlify(self.cipher_suite) + if self.cipher_suite is not None + else '' ), ) parts.append( 'Compression Method: %s' % ( - pretty_hexlify( - self.compression_method, - ) if self.compression_method is not None else '' + pretty_hexlify(self.compression_method) + if self.compression_method is not None + else '' ), ) parts.append( 'Extension: %s' % ( - pretty_hexlify( - self.extension, - ) if self.extension is not None else '' + pretty_hexlify(self.extension) + if self.extension is not None + else '' ), ) return os.linesep.join(parts) @@ -405,58 +406,57 @@ def parse(self, raw: bytes) -> Tuple[bool, bytes]: 'incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length, ) return False, raw - else: - # parse - self.msg_type = raw[0] - self.length = raw[1:4] - self.data = raw[: 4 + payload_length] - payload = raw[4: 4 + payload_length] - if self.msg_type == tlsHandshakeType.HELLO_REQUEST: - # parse hello request - self.hello_request = TlsHelloRequest() - self.hello_request.parse(payload) - elif self.msg_type == tlsHandshakeType.CLIENT_HELLO: - # parse client hello - self.client_hello = TlsClientHello() - self.client_hello.parse(payload) - elif self.msg_type == tlsHandshakeType.SERVER_HELLO: - # parse server hello - self.server_hello = TlsServerHello() - self.server_hello.parse(payload) - elif self.msg_type == tlsHandshakeType.CERTIFICATE: - # parse certificate - self.certificate = TlsCertificate() - self.certificate.parse(payload) - elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: - # parse server key exchange - self.server_key_exchange = TlsServerKeyExchange() - self.server_key_exchange.parse(payload) - elif self.msg_type == tlsHandshakeType.CERTIFICATE_REQUEST: - # parse certificate request - self.certificate_request = TlsCertificateRequest() - self.certificate_request.parse(payload) - elif self.msg_type == tlsHandshakeType.SERVER_HELLO_DONE: - # parse server hello done - self.server_hello_done = TlsServerHelloDone() - self.server_hello_done.parse(payload) - elif self.msg_type == tlsHandshakeType.CERTIFICATE_VERIFY: - # parse certificate verify - self.certificate_verify = TlsCertificateVerify() - self.certificate_verify.parse(payload) - elif self.msg_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE: - # parse client key exchange - self.client_key_exchange = TlsClientKeyExchange() - self.client_key_exchange.parse(payload) - elif self.msg_type == tlsHandshakeType.FINISHED: - # parse finished - self.finished = TlsFinished() - self.finished.parse(payload) - return True, raw[4 + payload_length:] + # parse + self.msg_type = raw[0] + self.length = raw[1:4] + self.data = raw[: 4 + payload_length] + payload = raw[4: 4 + payload_length] + if self.msg_type == tlsHandshakeType.HELLO_REQUEST: + # parse hello request + self.hello_request = TlsHelloRequest() + self.hello_request.parse(payload) + elif self.msg_type == tlsHandshakeType.CLIENT_HELLO: + # parse client hello + self.client_hello = TlsClientHello() + self.client_hello.parse(payload) + elif self.msg_type == tlsHandshakeType.SERVER_HELLO: + # parse server hello + self.server_hello = TlsServerHello() + self.server_hello.parse(payload) + elif self.msg_type == tlsHandshakeType.CERTIFICATE: + # parse certificate + self.certificate = TlsCertificate() + self.certificate.parse(payload) + elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: + # parse server key exchange + self.server_key_exchange = TlsServerKeyExchange() + self.server_key_exchange.parse(payload) + elif self.msg_type == tlsHandshakeType.CERTIFICATE_REQUEST: + # parse certificate request + self.certificate_request = TlsCertificateRequest() + self.certificate_request.parse(payload) + elif self.msg_type == tlsHandshakeType.SERVER_HELLO_DONE: + # parse server hello done + self.server_hello_done = TlsServerHelloDone() + self.server_hello_done.parse(payload) + elif self.msg_type == tlsHandshakeType.CERTIFICATE_VERIFY: + # parse certificate verify + self.certificate_verify = TlsCertificateVerify() + self.certificate_verify.parse(payload) + elif self.msg_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE: + # parse client key exchange + self.client_key_exchange = TlsClientKeyExchange() + self.client_key_exchange.parse(payload) + elif self.msg_type == tlsHandshakeType.FINISHED: + # parse finished + self.finished = TlsFinished() + self.finished.parse(payload) + return True, raw[4 + payload_length:] def build(self) -> bytes: - data = bytes() + data = b'' data += bytes([self.msg_type]) - payload = bytes() + payload = b'' if self.msg_type == tlsHandshakeType.CLIENT_HELLO: assert self.client_hello payload = self.client_hello.build() @@ -519,11 +519,11 @@ def parse(self, raw: bytes) -> Tuple[bool, bytes]: return True, raw[5 + payload_length:] def build(self) -> bytes: - data = bytes() + data = b'' data += bytes([self.content_type]) # ??? data += self.protocol_version - payload = bytes() + payload = b'' if self.content_type == tlsContentType.HANDSHAKE: assert self.handshake payload += self.handshake.build() diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 969f31671d..5975cf9196 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -302,8 +302,7 @@ async def read_from_descriptors(self, r: Readables) -> bool: if self.upstream.closed: logger.debug('Server closed connection, tearing down...') return self._close_and_release() - else: - return False + return False for plugin in self.plugins.values(): raw = plugin.handle_upstream_chunk(raw) diff --git a/tests/http/test_tls_parser.py b/tests/http/test_tls_parser.py index c6b99ab35f..79dd7a6f76 100644 --- a/tests/http/test_tls_parser.py +++ b/tests/http/test_tls_parser.py @@ -39,7 +39,7 @@ def test_parse_client_hello(self) -> None: assert tls.handshake self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.CLIENT_HELLO) self.assertEqual(tls.handshake.length, b'\x00\x00\xa1') - self.assertEqual(len(tls.handshake.build()), 0xa1 + 0x04) + self.assertEqual(len(tls.handshake.build()), 0xA1 + 0x04) assert tls.handshake.client_hello self.assertEqual( tls.handshake.client_hello.protocol_version, b'\x03\x03', From 0d3c6e3c8ea7b05a3751d07fb2503a542a3fbbb7 Mon Sep 17 00:00:00 2001 From: Jerry Date: Sat, 18 Dec 2021 19:40:01 +0800 Subject: [PATCH 08/14] bug fix in tls.py bug fix in tls.py add test data for tls parser --- tests/http/tls_server_hello.data | 337 +++++++++++++++++++++++++++++++ 1 file changed, 337 insertions(+) create mode 100644 tests/http/tls_server_hello.data diff --git a/tests/http/tls_server_hello.data b/tests/http/tls_server_hello.data new file mode 100644 index 0000000000..72553c3153 --- /dev/null +++ b/tests/http/tls_server_hello.data @@ -0,0 +1,337 @@ +16 03 03 00 59 02 00 00 55 03 03 60 f9 54 e7 1b +c2 60 2d 02 94 ed 2a 82 d2 1d 06 71 3c 52 d1 23 +0e 73 fe c9 93 8a 05 71 f5 48 46 20 b0 d9 d0 1c +bd 6b 49 27 47 88 14 c6 ef 04 bc 48 3b 01 5f 00 +61 ae f8 a2 45 5e 7b fe fb ed f4 a8 c0 2f 00 00 +0d ff 01 00 01 00 00 0b 00 04 03 00 01 02 16 03 +03 13 4f 0b 00 13 4b 00 13 48 00 05 e0 30 82 05 +dc 30 82 04 c4 a0 03 02 01 02 02 10 01 58 f3 0a +d5 9d 62 af 20 9f 15 ee 4f c2 bb 33 30 0d 06 09 +2a 86 48 86 f7 0d 01 01 0b 05 00 30 46 31 0b 30 +09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06 03 +55 04 0a 13 06 41 6d 61 7a 6f 6e 31 15 30 13 06 +03 55 04 0b 13 0c 53 65 72 76 65 72 20 43 41 20 +31 42 31 0f 30 0d 06 03 55 04 03 13 06 41 6d 61 +7a 6f 6e 30 1e 17 0d 32 31 31 31 32 31 30 30 30 +30 30 30 5a 17 0d 32 32 31 32 31 39 32 33 35 39 +35 39 5a 30 16 31 14 30 12 06 03 55 04 03 13 0b +68 74 74 70 62 69 6e 2e 6f 72 67 30 82 01 22 30 +0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 82 +01 0f 00 30 82 01 0a 02 82 01 01 00 84 e4 27 a5 +ec eb c0 0d 2f 1f 37 f8 ec f6 be 3b ce 1f 5a e7 +bf e7 ad 93 a9 0a d5 8a bb 5f fb 77 ec 19 07 77 +32 6c 27 21 df a4 b0 01 90 bd 63 78 33 5f e1 49 +e9 58 25 bd b8 ea 91 6a 18 1e ed f9 92 3e be 30 +58 1f 8f c9 e8 28 85 b0 ac 33 ba ca a4 48 5e b7 +7e d4 21 f0 ec 74 39 ab 01 f4 7b d1 94 e3 c7 4f +f0 79 0e c9 21 f0 73 62 27 ed 65 13 14 99 11 a3 +bd ce c4 3d c0 ee b6 6e c1 87 b0 c5 ff 50 fd d5 +40 1c 97 2d 50 59 bb 40 cc 2a c9 76 84 66 94 37 +2f 7e 2e 06 6f e3 f8 95 a5 60 cf 43 5f a4 86 97 +f1 3f dc 75 1f e9 8c 95 45 81 81 83 ea ee 31 b9 +71 e4 db 96 d2 28 17 fb 14 40 f3 51 fb 22 88 05 +29 ee 88 84 97 14 73 bd 1c 49 e4 e6 98 7a d7 48 +56 99 54 0a 22 b0 80 3e c0 14 30 cf 66 eb e8 83 +6f ae 0d 8e b3 23 59 5f db 92 6d fa af e1 41 33 +3f 4b f0 77 c4 07 88 71 97 0c 20 e5 02 03 01 00 +01 a3 82 02 f4 30 82 02 f0 30 1f 06 03 55 1d 23 +04 18 30 16 80 14 59 a4 66 06 52 a0 7b 95 92 3c +a3 94 07 27 96 74 5b f9 3d d0 30 1d 06 03 55 1d +0e 04 16 04 14 cf 07 84 b8 03 48 ac c9 ac db 11 +65 0a 7d 29 ff d6 97 4b b3 30 25 06 03 55 1d 11 +04 1e 30 1c 82 0b 68 74 74 70 62 69 6e 2e 6f 72 +67 82 0d 2a 2e 68 74 74 70 62 69 6e 2e 6f 72 67 +30 0e 06 03 55 1d 0f 01 01 ff 04 04 03 02 05 a0 +30 1d 06 03 55 1d 25 04 16 30 14 06 08 2b 06 01 +05 05 07 03 01 06 08 2b 06 01 05 05 07 03 02 30 +3d 06 03 55 1d 1f 04 36 30 34 30 32 a0 30 a0 2e +86 2c 68 74 74 70 3a 2f 2f 63 72 6c 2e 73 63 61 +31 62 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63 +6f 6d 2f 73 63 61 31 62 2d 31 2e 63 72 6c 30 13 +06 03 55 1d 20 04 0c 30 0a 30 08 06 06 67 81 0c +01 02 01 30 75 06 08 2b 06 01 05 05 07 01 01 04 +69 30 67 30 2d 06 08 2b 06 01 05 05 07 30 01 86 +21 68 74 74 70 3a 2f 2f 6f 63 73 70 2e 73 63 61 +31 62 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63 +6f 6d 30 36 06 08 2b 06 01 05 05 07 30 02 86 2a +68 74 74 70 3a 2f 2f 63 72 74 2e 73 63 61 31 62 +2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63 6f 6d +2f 73 63 61 31 62 2e 63 72 74 30 0c 06 03 55 1d +13 01 01 ff 04 02 30 00 30 82 01 7d 06 0a 2b 06 +01 04 01 d6 79 02 04 02 04 82 01 6d 04 82 01 69 +01 67 00 76 00 29 79 be f0 9e 39 39 21 f0 56 73 +9f 63 a5 77 e5 be 57 7d 9c 60 0a f8 f9 4d 5d 26 +5c 25 5d c7 84 00 00 01 7d 40 73 08 b5 00 00 04 +03 00 47 30 45 02 20 5d d8 f1 cc 14 99 9f 5c 99 +84 9c 94 da d2 60 e8 58 0d e0 b3 49 30 18 1f 0a +48 86 9e a8 00 72 81 02 21 00 ff c9 ca c0 dd 06 +dd 2c 14 89 2d ff 35 dc e5 56 eb 9a 92 be 1a ee +8b e2 03 33 7b cd 78 92 9b 37 00 76 00 51 a3 b0 +f5 fd 01 79 9c 56 6d b8 37 78 8f 0c a4 7a cc 1b +27 cb f7 9e 88 42 9a 0d fe d4 8b 05 e5 00 00 01 +7d 40 73 08 c9 00 00 04 03 00 47 30 45 02 20 7e +29 67 be 79 d6 34 0e 27 f9 55 db 4b 03 28 d0 96 +38 b7 6d f9 c4 b6 a6 35 08 14 4a 48 f2 7c 25 02 +21 00 8f 68 71 a6 2b c8 3e d5 fd b5 17 84 70 90 +9a 02 e0 b9 1a 1f 74 99 9c fd 95 64 32 ba c2 03 +d1 54 00 75 00 df a5 5e ab 68 82 4f 1f 6c ad ee +b8 5f 4e 3e 5a ea cd a2 12 a4 6a 5e 8e 3b 12 c0 +20 44 5c 2a 73 00 00 01 7d 40 73 08 a2 00 00 04 +03 00 46 30 44 02 20 4c 9e 68 c3 ac cb 92 d2 b0 +a9 08 a1 dd 82 7c 13 0b dc 9f fe 0b 08 ef 25 e9 +1d 30 12 38 41 82 36 02 20 18 29 da 43 a6 d0 df +b5 71 bb 19 23 b3 f1 28 29 a9 1b 54 a1 62 07 b3 +5c 28 4a 3e ee c0 42 70 bb 30 0d 06 09 2a 86 48 +86 f7 0d 01 01 0b 05 00 03 82 01 01 00 2f 64 1d +d3 7d 3c 06 41 7b 6a 1c 94 60 99 31 92 b7 eb e1 +6c b2 ac ee d2 5b f4 ec 36 94 a6 c6 a3 c5 f8 3f +d0 a9 07 01 b7 cb 4f 1d 68 35 a0 d1 1c 51 88 a7 +97 9a ad 03 89 32 1e aa 40 96 0c d8 97 4e 05 52 +94 a1 90 d0 e3 73 07 75 1a 29 20 98 d3 08 85 21 +b1 dc 41 da 41 ec 87 be b2 88 80 24 5e 4c ad 5f +3c fc 43 bc 55 36 6a 8f 67 9c 47 e2 43 43 a8 b2 +c4 55 0b a1 1a 26 c4 3e fc 21 27 4e 2a f4 04 43 +e3 59 18 d6 d8 2f e0 f5 83 87 47 23 fd 3c ee 11 +ff 9f 0a 3b 2f 29 74 96 70 5a 43 b2 18 ac a5 65 +e9 ac 2f 2e ac 47 c8 77 50 4d 0a 5b 44 da c0 cc +06 3b eb 23 59 4d 9e 07 fb 8e 27 a6 1c 68 04 b4 +a2 0d 52 ec da cb 0e a4 f7 1d 45 92 7f cd 60 ee +a9 4d 78 2d e1 36 02 3c 7c cc 45 2c f4 e4 86 7f +3e bf 6d b4 48 98 a9 8b a8 a0 9e 42 51 54 5a 02 +d5 a4 c5 a2 55 cc ab fa 0c a9 1a 94 1c 00 04 4d +30 82 04 49 30 82 03 31 a0 03 02 01 02 02 13 06 +7f 94 57 85 87 e8 ac 77 de b2 53 32 5b bc 99 8b +56 0d 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 +00 30 39 31 0b 30 09 06 03 55 04 06 13 02 55 53 +31 0f 30 0d 06 03 55 04 0a 13 06 41 6d 61 7a 6f +6e 31 19 30 17 06 03 55 04 03 13 10 41 6d 61 7a +6f 6e 20 52 6f 6f 74 20 43 41 20 31 30 1e 17 0d +31 35 31 30 32 32 30 30 30 30 30 30 5a 17 0d 32 +35 31 30 31 39 30 30 30 30 30 30 5a 30 46 31 0b +30 09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06 +03 55 04 0a 13 06 41 6d 61 7a 6f 6e 31 15 30 13 +06 03 55 04 0b 13 0c 53 65 72 76 65 72 20 43 41 +20 31 42 31 0f 30 0d 06 03 55 04 03 13 06 41 6d +61 7a 6f 6e 30 82 01 22 30 0d 06 09 2a 86 48 86 +f7 0d 01 01 01 05 00 03 82 01 0f 00 30 82 01 0a +02 82 01 01 00 c2 4e 16 67 dd ce bc 6a c8 37 5a +ec 3a 30 b0 1d e6 d1 12 e8 12 28 48 cc e8 29 c1 +b9 6e 53 d5 a3 eb 03 39 1a cc 77 87 f6 01 b9 d9 +70 cc cf 6b 8d e3 e3 03 71 86 99 6d cb a6 94 2a +4e 13 d6 a7 bd 04 ec 0a 16 3c 0a eb 39 b1 c4 b5 +58 a3 b6 c7 56 25 ec 3e 52 7a a8 e3 29 16 07 b9 +6e 50 cf fb 5f 31 f8 1d ba 03 4a 62 89 03 ae 3e +47 f2 0f 27 91 e3 14 20 85 f8 fa e9 8a 35 f5 5f +9e 99 4d e7 6b 37 ef a4 50 3e 44 ec fa 5a 85 66 +07 9c 7e 17 6a 55 f3 17 8a 35 1e ee e9 ac c3 75 +4e 58 55 7d 53 6b 0a 6b 9b 14 42 d7 e5 ac 01 89 +b3 ea a3 fe cf c0 2b 0c 84 c2 d8 53 15 cb 67 f0 +d0 88 ca 3a d1 17 73 f5 5f 9a d4 c5 72 1e 7e 01 +f1 98 30 63 2a aa f2 7a 2d c5 e2 02 1a 86 e5 32 +3e 0e bd 11 b4 cf 3c 93 ef 17 50 10 9e 43 c2 06 +2a e0 0d 68 be d3 88 8b 4a 65 8c 4a d4 c3 2e 4c +9b 55 f4 86 e5 02 03 01 00 01 a3 82 01 3b 30 82 +01 37 30 12 06 03 55 1d 13 01 01 ff 04 08 30 06 +01 01 ff 02 01 00 30 0e 06 03 55 1d 0f 01 01 ff +04 04 03 02 01 86 30 1d 06 03 55 1d 0e 04 16 04 +14 59 a4 66 06 52 a0 7b 95 92 3c a3 94 07 27 96 +74 5b f9 3d d0 30 1f 06 03 55 1d 23 04 18 30 16 +80 14 84 18 cc 85 34 ec bc 0c 94 94 2e 08 59 9c +c7 b2 10 4e 0a 08 30 7b 06 08 2b 06 01 05 05 07 +01 01 04 6f 30 6d 30 2f 06 08 2b 06 01 05 05 07 +30 01 86 23 68 74 74 70 3a 2f 2f 6f 63 73 70 2e +72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 72 +75 73 74 2e 63 6f 6d 30 3a 06 08 2b 06 01 05 05 +07 30 02 86 2e 68 74 74 70 3a 2f 2f 63 72 74 2e +72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 72 +75 73 74 2e 63 6f 6d 2f 72 6f 6f 74 63 61 31 2e +63 65 72 30 3f 06 03 55 1d 1f 04 38 30 36 30 34 +a0 32 a0 30 86 2e 68 74 74 70 3a 2f 2f 63 72 6c +2e 72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 +72 75 73 74 2e 63 6f 6d 2f 72 6f 6f 74 63 61 31 +2e 63 72 6c 30 13 06 03 55 1d 20 04 0c 30 0a 30 +08 06 06 67 81 0c 01 02 01 30 0d 06 09 2a 86 48 +86 f7 0d 01 01 0b 05 00 03 82 01 01 00 85 92 be +35 bb 79 cf a3 81 42 1c e4 e3 63 73 53 39 52 35 +e7 d1 ad fd ae 99 8a ac 89 12 2f bb e7 6f 9a d5 +4e 72 ea 20 30 61 f9 97 b2 cd a5 27 02 45 a8 ca +76 3e 98 4a 83 9e b6 e6 45 e0 f2 43 f6 08 de 6d +e8 6e db 31 07 13 f0 2f 31 0d 93 6d 61 37 7b 58 +f0 fc 51 98 91 28 02 4f 05 76 b7 d3 f0 1b c2 e6 +5e d0 66 85 11 0f 2e 81 c6 10 81 29 fe 20 60 48 +f3 f2 f0 84 13 53 65 35 15 11 6b 82 51 40 55 57 +5f 18 b5 b0 22 3e ad f2 5e a3 01 e3 c3 b3 f9 cb +41 5a e6 52 91 bb e4 36 87 4f 2d a9 a4 07 68 35 +ba 94 72 cd 0e ea 0e 7d 57 f2 79 fc 37 c5 7b 60 +9e b2 eb c0 2d 90 77 0d 49 10 27 a5 38 ad c4 12 +a3 b4 a3 c8 48 b3 15 0b 1e e2 e2 19 dc c4 76 52 +c8 bc 8a 41 78 70 d9 6d 97 b3 4a 8b 78 2d 5e b4 +0f a3 4c 60 ca e1 47 cb 78 2d 12 17 b1 52 8b ca +39 2c bd b5 2f c2 33 02 96 ab da 94 7f 00 04 96 +30 82 04 92 30 82 03 7a a0 03 02 01 02 02 13 06 +7f 94 4a 2a 27 cd f3 fa c2 ae 2b 01 f9 08 ee b9 +c4 c6 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 +00 30 81 98 31 0b 30 09 06 03 55 04 06 13 02 55 +53 31 10 30 0e 06 03 55 04 08 13 07 41 72 69 7a +6f 6e 61 31 13 30 11 06 03 55 04 07 13 0a 53 63 +6f 74 74 73 64 61 6c 65 31 25 30 23 06 03 55 04 +0a 13 1c 53 74 61 72 66 69 65 6c 64 20 54 65 63 +68 6e 6f 6c 6f 67 69 65 73 2c 20 49 6e 63 2e 31 +3b 30 39 06 03 55 04 03 13 32 53 74 61 72 66 69 +65 6c 64 20 53 65 72 76 69 63 65 73 20 52 6f 6f +74 20 43 65 72 74 69 66 69 63 61 74 65 20 41 75 +74 68 6f 72 69 74 79 20 2d 20 47 32 30 1e 17 0d +31 35 30 35 32 35 31 32 30 30 30 30 5a 17 0d 33 +37 31 32 33 31 30 31 30 30 30 30 5a 30 39 31 0b +30 09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06 +03 55 04 0a 13 06 41 6d 61 7a 6f 6e 31 19 30 17 +06 03 55 04 03 13 10 41 6d 61 7a 6f 6e 20 52 6f +6f 74 20 43 41 20 31 30 82 01 22 30 0d 06 09 2a +86 48 86 f7 0d 01 01 01 05 00 03 82 01 0f 00 30 +82 01 0a 02 82 01 01 00 b2 78 80 71 ca 78 d5 e3 +71 af 47 80 50 74 7d 6e d8 d7 88 76 f4 99 68 f7 +58 21 60 f9 74 84 01 2f ac 02 2d 86 d3 a0 43 7a +4e b2 a4 d0 36 ba 01 be 8d db 48 c8 07 17 36 4c +f4 ee 88 23 c7 3e eb 37 f5 b5 19 f8 49 68 b0 de +d7 b9 76 38 1d 61 9e a4 fe 82 36 a5 e5 4a 56 e4 +45 e1 f9 fd b4 16 fa 74 da 9c 9b 35 39 2f fa b0 +20 50 06 6c 7a d0 80 b2 a6 f9 af ec 47 19 8f 50 +38 07 dc a2 87 39 58 f8 ba d5 a9 f9 48 67 30 96 +ee 94 78 5e 6f 89 a3 51 c0 30 86 66 a1 45 66 ba +54 eb a3 c3 91 f9 48 dc ff d1 e8 30 2d 7d 2d 74 +70 35 d7 88 24 f7 9e c4 59 6e bb 73 87 17 f2 32 +46 28 b8 43 fa b7 1d aa ca b4 f2 9f 24 0e 2d 4b +f7 71 5c 5e 69 ff ea 95 02 cb 38 8a ae 50 38 6f +db fb 2d 62 1b c5 c7 1e 54 e1 77 e0 67 c8 0f 9c +87 23 d6 3f 40 20 7f 20 80 c4 80 4c 3e 3b 24 26 +8e 04 ae 6c 9a c8 aa 0d 02 03 01 00 01 a3 82 01 +31 30 82 01 2d 30 0f 06 03 55 1d 13 01 01 ff 04 +05 30 03 01 01 ff 30 0e 06 03 55 1d 0f 01 01 ff +04 04 03 02 01 86 30 1d 06 03 55 1d 0e 04 16 04 +14 84 18 cc 85 34 ec bc 0c 94 94 2e 08 59 9c c7 +b2 10 4e 0a 08 30 1f 06 03 55 1d 23 04 18 30 16 +80 14 9c 5f 00 df aa 01 d7 30 2b 38 88 a2 b8 6d +4a 9c f2 11 91 83 30 78 06 08 2b 06 01 05 05 07 +01 01 04 6c 30 6a 30 2e 06 08 2b 06 01 05 05 07 +30 01 86 22 68 74 74 70 3a 2f 2f 6f 63 73 70 2e +72 6f 6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 +73 74 2e 63 6f 6d 30 38 06 08 2b 06 01 05 05 07 +30 02 86 2c 68 74 74 70 3a 2f 2f 63 72 74 2e 72 +6f 6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 73 +74 2e 63 6f 6d 2f 72 6f 6f 74 67 32 2e 63 65 72 +30 3d 06 03 55 1d 1f 04 36 30 34 30 32 a0 30 a0 +2e 86 2c 68 74 74 70 3a 2f 2f 63 72 6c 2e 72 6f +6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 +2e 63 6f 6d 2f 72 6f 6f 74 67 32 2e 63 72 6c 30 +11 06 03 55 1d 20 04 0a 30 08 30 06 06 04 55 1d +20 00 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 +00 03 82 01 01 00 62 37 42 5c bc 10 b5 3e 8b 2c +e9 0c 9b 6c 45 e2 07 00 7a f9 c5 58 0b b9 08 8c +3e ed b3 25 3c b5 6f 50 e4 cd 35 6a a7 93 34 96 +32 21 a9 48 44 ab 9c ed 3d b4 aa 73 6d e4 7f 16 +80 89 6c cf 28 03 18 83 47 79 a3 10 7e 30 5b ac +3b b0 60 e0 77 d4 08 a6 e1 1d 7c 5e c0 bb f9 9a +7b 22 9d a7 00 09 7e ac 46 17 83 dc 9c 26 57 99 +30 39 62 96 8f ed da de aa c5 cc 1b 3e ca 43 68 +6c 57 16 bc d5 0e 20 2e fe ff c2 6a 5d 2e a0 4a +6d 14 58 87 94 e6 39 31 5f 7c 73 cb 90 88 6a 84 +11 96 27 a6 ed d9 81 46 a6 7e a3 72 00 0a 52 3e +83 88 07 63 77 89 69 17 0f 39 85 d2 ab 08 45 4d +d0 51 3a fd 5d 5d 37 64 4c 7e 30 b2 55 24 42 9d +36 b0 5d 9c 17 81 61 f1 ca f9 10 02 24 ab eb 0d +74 91 8d 7b 45 29 50 39 88 b2 a6 89 35 25 1e 14 +6a 47 23 31 2f 5c 9a fa ad 9a 0e 62 51 a4 2a a9 +c4 f9 34 9d 21 18 00 04 79 30 82 04 75 30 82 03 +5d a0 03 02 01 02 02 09 00 a7 0e 4a 4c 34 82 b7 +7f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 +30 68 31 0b 30 09 06 03 55 04 06 13 02 55 53 31 +25 30 23 06 03 55 04 0a 13 1c 53 74 61 72 66 69 +65 6c 64 20 54 65 63 68 6e 6f 6c 6f 67 69 65 73 +2c 20 49 6e 63 2e 31 32 30 30 06 03 55 04 0b 13 +29 53 74 61 72 66 69 65 6c 64 20 43 6c 61 73 73 +20 32 20 43 65 72 74 69 66 69 63 61 74 69 6f 6e +20 41 75 74 68 6f 72 69 74 79 30 1e 17 0d 30 39 +30 39 30 32 30 30 30 30 30 30 5a 17 0d 33 34 30 +36 32 38 31 37 33 39 31 36 5a 30 81 98 31 0b 30 +09 06 03 55 04 06 13 02 55 53 31 10 30 0e 06 03 +55 04 08 13 07 41 72 69 7a 6f 6e 61 31 13 30 11 +06 03 55 04 07 13 0a 53 63 6f 74 74 73 64 61 6c +65 31 25 30 23 06 03 55 04 0a 13 1c 53 74 61 72 +66 69 65 6c 64 20 54 65 63 68 6e 6f 6c 6f 67 69 +65 73 2c 20 49 6e 63 2e 31 3b 30 39 06 03 55 04 +03 13 32 53 74 61 72 66 69 65 6c 64 20 53 65 72 +76 69 63 65 73 20 52 6f 6f 74 20 43 65 72 74 69 +66 69 63 61 74 65 20 41 75 74 68 6f 72 69 74 79 +20 2d 20 47 32 30 82 01 22 30 0d 06 09 2a 86 48 +86 f7 0d 01 01 01 05 00 03 82 01 0f 00 30 82 01 +0a 02 82 01 01 00 d5 0c 3a c4 2a f9 4e e2 f5 be +19 97 5f 8e 88 53 b1 1f 3f cb cf 9f 20 13 6d 29 +3a c8 0f 7d 3c f7 6b 76 38 63 d9 36 60 a8 9b 5e +5c 00 80 b2 2f 59 7f f6 87 f9 25 43 86 e7 69 1b +52 9a 90 e1 71 e3 d8 2d 0d 4e 6f f6 c8 49 d9 b6 +f3 1a 56 ae 2b b6 74 14 eb cf fb 26 e3 1a ba 1d +96 2e 6a 3b 58 94 89 47 56 ff 25 a0 93 70 53 83 +da 84 74 14 c3 67 9e 04 68 3a df 8e 40 5a 1d 4a +4e cf 43 91 3b e7 56 d6 00 70 cb 52 ee 7b 7d ae +3a e7 bc 31 f9 45 f6 c2 60 cf 13 59 02 2b 80 cc +34 47 df b9 de 90 65 6d 02 cf 2c 91 a6 a6 e7 de +85 18 49 7c 66 4e a3 3a 6d a9 b5 ee 34 2e ba 0d +03 b8 33 df 47 eb b1 6b 8d 25 d9 9b ce 81 d1 45 +46 32 96 70 87 de 02 0e 49 43 85 b6 6c 73 bb 64 +ea 61 41 ac c9 d4 54 df 87 2f c7 22 b2 26 cc 9f +59 54 68 9f fc be 2a 2f c4 55 1c 75 40 60 17 85 +02 55 39 8b 7f 05 02 03 01 00 01 a3 81 f0 30 81 +ed 30 0f 06 03 55 1d 13 01 01 ff 04 05 30 03 01 +01 ff 30 0e 06 03 55 1d 0f 01 01 ff 04 04 03 02 +01 86 30 1d 06 03 55 1d 0e 04 16 04 14 9c 5f 00 +df aa 01 d7 30 2b 38 88 a2 b8 6d 4a 9c f2 11 91 +83 30 1f 06 03 55 1d 23 04 18 30 16 80 14 bf 5f +b7 d1 ce dd 1f 86 f4 5b 55 ac dc d7 10 c2 0e a9 +88 e7 30 4f 06 08 2b 06 01 05 05 07 01 01 04 43 +30 41 30 1c 06 08 2b 06 01 05 05 07 30 01 86 10 +68 74 74 70 3a 2f 2f 6f 2e 73 73 32 2e 75 73 2f +30 21 06 08 2b 06 01 05 05 07 30 02 86 15 68 74 +74 70 3a 2f 2f 78 2e 73 73 32 2e 75 73 2f 78 2e +63 65 72 30 26 06 03 55 1d 1f 04 1f 30 1d 30 1b +a0 19 a0 17 86 15 68 74 74 70 3a 2f 2f 73 2e 73 +73 32 2e 75 73 2f 72 2e 63 72 6c 30 11 06 03 55 +1d 20 04 0a 30 08 30 06 06 04 55 1d 20 00 30 0d +06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 03 82 01 +01 00 23 1d e3 8a 57 ca 7d e9 17 79 4c f1 1e 55 +fd cc 53 6e 3e 47 0f df c6 55 f2 b2 04 36 ed 80 +1f 53 c4 5d 34 28 6b be c7 55 fc 67 ea cb 3f 7f +90 b2 33 cd 1b 58 10 82 02 f8 f8 2f f5 13 60 d4 +05 ce f1 81 08 c1 dd a7 75 97 4f 18 b9 6d de f7 +93 91 08 ba 7e 40 2c ed c1 ea bb 76 9e 33 06 77 +1d 0d 08 7f 53 dd 1b 64 ab 82 27 f1 69 d5 4d 5e +ae f4 a1 c3 75 a7 58 44 2d f2 3c 70 98 ac ba 69 +b6 95 77 7f 0f 31 5e 2c fc a0 87 3a 47 69 f0 79 +5f f4 14 54 a4 95 5e 11 78 12 60 27 ce 9f c2 77 +ff 23 53 77 5d ba ff ea 59 e7 db cf af 92 96 ef +24 9a 35 10 7a 9c 91 c6 0e 7d 99 f6 3f 19 df f5 +72 54 e1 15 a9 07 59 7b 83 bf 52 2e 46 8c b2 00 +64 76 1c 48 d3 d8 79 e8 6e 56 cc ae 2c 03 90 d7 +19 38 99 e4 ca 09 19 5b ff 07 96 b0 a8 7f 34 49 +df 56 a9 f7 b0 5f ed 33 ed 8c 47 b7 30 03 5d f4 +03 8c 16 03 03 01 4d 0c 00 01 49 03 00 17 41 04 +e4 b3 e7 a8 fe ec e3 99 76 95 8d a5 32 ce de bd +ce 8c 2b 3d 31 f4 97 88 9c 39 e5 a0 1f 37 ec f9 +3d b1 e8 5c 5e a2 da 02 6d 91 e5 41 51 ca 72 6e +68 61 77 c8 3a e6 c1 d2 84 c7 4b 64 de 05 06 70 +06 01 01 00 28 3b e1 f9 23 fd 39 af 3a fa d6 f2 +ad e9 92 ae 66 49 90 58 36 19 2d 21 7b 0b 71 48 +ef 61 3f fe a0 db 3c 61 70 7f 3f cf 14 b5 a9 a8 +5b fa 45 45 8c f2 3f 13 5e da 19 f4 91 dd 2d f1 +8a 29 f6 7a 04 66 d7 54 14 38 b6 19 9d 07 66 a0 +1d c2 a5 cc ac a4 50 f1 d0 71 7a 75 84 a5 87 bd +7b f6 3d 75 6f 66 0f 28 0f 72 17 15 1b a4 fc 16 +9f 97 03 17 3a 88 a9 62 ce bc f3 cf 16 e1 e6 70 +d3 ad 65 dc fb 25 55 88 40 bf 5f b7 35 ac 4b 48 +9e f8 6b ab 6e b7 4d 10 41 a9 c8 19 d2 d7 76 51 +c9 d6 49 92 10 7e 7a 52 d8 ba 25 44 d3 4a 59 56 +15 49 aa 12 b0 ea 56 dc cf b1 43 07 ca cd 61 49 +e3 46 19 a8 1b 24 34 e5 c8 5f a5 cd 81 08 de b6 +6e 19 85 19 fc f0 d5 d1 98 19 1d 1a 09 fc d2 36 +87 c0 19 70 fc ca 1d 37 dc bd 1d 33 c3 8d 6f 2d +5d 29 4a ec 7f f8 f5 cf ea c8 de 47 8f 6f 17 42 +c4 23 4f 76 16 03 03 00 04 0e 00 00 00 \ No newline at end of file From e1c017e3248042b8353303990c3c289ef0787dc0 Mon Sep 17 00:00:00 2001 From: Jerry Date: Sat, 18 Dec 2021 20:06:45 +0800 Subject: [PATCH 09/14] bug fix in tls parser bug fix in tls parser --- proxy/http/parser/tls.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/proxy/http/parser/tls.py b/proxy/http/parser/tls.py index 6b12eb93e2..a0ddb24b0a 100644 --- a/proxy/http/parser/tls.py +++ b/proxy/http/parser/tls.py @@ -378,11 +378,11 @@ class TlsParser: def __init__(self) -> None: self.content_type: int = tlsContentType.OTHER - self.protocol_version: Optional[TlsProtocolVersion] = None + self.protocol_version: Optional[bytes] = None self.length: Optional[bytes] = None # only parse hand shake payload temporary self.handshake: Optional[TlsHandshake] = None - self.certificate: Optional[TlsCertificate] + self.certificate: Optional[TlsCertificate] = None def parse(self, raw: bytes) -> (bool, bytes): """parse TLS fragmentation @@ -396,7 +396,6 @@ def parse(self, raw: bytes) -> (bool, bytes): return False, raw else: payload_length, = struct.unpack('!H', raw[3:5]) - self.protocol_version if length < 5 + payload_length: logger.debug('incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length) return False, raw From 8157dbe4740d0bfd3a3dd49a6ede2d36b911090d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 12:19:55 +0000 Subject: [PATCH 10/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/http/tls_server_hello.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/http/tls_server_hello.data b/tests/http/tls_server_hello.data index 72553c3153..a22f74ad4a 100644 --- a/tests/http/tls_server_hello.data +++ b/tests/http/tls_server_hello.data @@ -334,4 +334,4 @@ e3 46 19 a8 1b 24 34 e5 c8 5f a5 cd 81 08 de b6 6e 19 85 19 fc f0 d5 d1 98 19 1d 1a 09 fc d2 36 87 c0 19 70 fc ca 1d 37 dc bd 1d 33 c3 8d 6f 2d 5d 29 4a ec 7f f8 f5 cf ea c8 de 47 8f 6f 17 42 -c4 23 4f 76 16 03 03 00 04 0e 00 00 00 \ No newline at end of file +c4 23 4f 76 16 03 03 00 04 0e 00 00 00 From 8937c4b8725514c5b9dc4ac17c0f48256b856552 Mon Sep 17 00:00:00 2001 From: Jerry Date: Tue, 21 Dec 2021 11:12:52 +0800 Subject: [PATCH 11/14] remove unnecessary tls parser remove unnecessary tls parser and related tests --- proxy/http/parser/tls.py | 532 ------------------------------- tests/http/test_tls_parser.py | 69 ---- tests/http/tls_server_hello.data | 337 -------------------- 3 files changed, 938 deletions(-) delete mode 100644 proxy/http/parser/tls.py delete mode 100644 tests/http/test_tls_parser.py delete mode 100644 tests/http/tls_server_hello.data diff --git a/proxy/http/parser/tls.py b/proxy/http/parser/tls.py deleted file mode 100644 index 8d8cfe7300..0000000000 --- a/proxy/http/parser/tls.py +++ /dev/null @@ -1,532 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import os -import binascii -import struct -import logging - -from typing import NamedTuple, Optional, Tuple - - -logger = logging.getLogger(__name__) - - -def pretty_hexlify(raw: bytes) -> str: - hexlified = binascii.hexlify(raw).decode('utf-8') - return ' '.join([hexlified[i: i+2] for i in range(0, len(hexlified), 2)]) - - -TlsContentType = NamedTuple( - 'TlsContentType', [ - ('CHANGE_CIPHER_SPEC', int), - ('ALERT', int), - ('HANDSHAKE', int), - ('APPLICATION_DATA', int), - ('OTHER', int), - ], -) -tlsContentType = TlsContentType(20, 21, 22, 23, 255) - - -class TlsProtocolVersion: - """Protocol Version""" - - def __init__(self) -> None: - self.major = 0 - self.minor = 0 - - def set_value(self, major: int, minor: int) -> None: - self.major = major - self.minor = minor - - -TlsHandshakeType = NamedTuple( - 'TlsHandshakeType', [ - ('HELLO_REQUEST', int), - ('CLIENT_HELLO', int), - ('SERVER_HELLO', int), - ('CERTIFICATE', int), - ('SERVER_KEY_EXCHANGE', int), - ('CERTIFICATE_REQUEST', int), - ('SERVER_HELLO_DONE', int), - ('CERTIFICATE_VERIFY', int), - ('CLIENT_KEY_EXCHANGE', int), - ('FINISHED', int), - ('OTHER', int), - ], -) -tlsHandshakeType = TlsHandshakeType(0, 1, 2, 11, 12, 13, 14, 15, 16, 20, 255) - - -class TlsHelloRequest: - """TLS Hello Request""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> None: - self.data = raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsClientHello: - """TLS Client Hello""" - - def __init__(self) -> None: - self.protocol_version: Optional[bytes] = None - self.random: Optional[bytes] = None - self.session_id: Optional[bytes] = None - self.cipher_suite: Optional[bytes] = None - self.compression_method: Optional[bytes] = None - self.extension: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - try: - idx = 0 - length = len(raw) - self.protocol_version = raw[idx:idx + 2] - idx += 2 - self.random = raw[idx:idx + 32] - idx += 32 - session_length = raw[idx] - self.session_id = raw[idx: idx + 1 + session_length] - idx += 1 + session_length - cipher_suite_length, = struct.unpack('!H', raw[idx: idx + 2]) - self.cipher_suite = raw[idx: idx + 2 + cipher_suite_length] - idx += 2 + cipher_suite_length - compression_method_length = raw[idx] - self.compression_method = raw[ - idx: idx + - 1 + compression_method_length - ] - idx += 1 + compression_method_length - # extension - if idx == length: - self.extension = b'' - else: - extension_length, = struct.unpack('!H', raw[idx: idx + 2]) - self.extension = raw[idx: idx + 2 + extension_length] - idx += 2 + extension_length - return True, raw[idx:] - except Exception as e: - logger.exception(e) - return False, raw - - def build(self) -> bytes: - # calculate length - return b''.join([ - bs for bs in ( - self.protocol_version, self.random, self.session_id, self.cipher_suite, - self.compression_method, self.extension, - ) if bs is not None - ]) - - def format(self) -> str: - parts = [] - parts.append( - 'Protocol Version: %s' % ( - pretty_hexlify(self.protocol_version) - if self.protocol_version is not None - else '' - ), - ) - parts.append( - 'Random: %s' % ( - pretty_hexlify(self.random) - if self.random is not None else '' - ), - ) - parts.append( - 'Session ID: %s' % ( - pretty_hexlify(self.session_id) - if self.session_id is not None - else '' - ), - ) - parts.append( - 'Cipher Suite: %s' % ( - pretty_hexlify(self.cipher_suite) - if self.cipher_suite is not None - else '' - ), - ) - parts.append( - 'Compression Method: %s' % ( - pretty_hexlify(self.compression_method) - if self.compression_method is not None - else '' - ), - ) - parts.append( - 'Extension: %s' % ( - pretty_hexlify(self.extension) - if self.extension is not None - else '' - ), - ) - return os.linesep.join(parts) - - -class TlsServerHello: - """TLS Server Hello""" - - def __init__(self) -> None: - self.protocol_version: Optional[bytes] = None - self.random: Optional[bytes] = None - self.session_id: Optional[bytes] = None - self.cipher_suite: Optional[bytes] = None - self.compression_method: Optional[bytes] = None - self.extension: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - try: - idx = 0 - length = len(raw) - self.protocol_version = raw[idx:idx + 2] - idx += 2 - self.random = raw[idx:idx + 32] - idx += 32 - session_length = raw[idx] - self.session_id = raw[idx: idx + 1 + session_length] - idx += 1 + session_length - self.cipher_suite = raw[idx: idx + 2] - idx += 2 - compression_method_length = raw[idx] - self.compression_method = raw[ - idx: idx + - 1 + compression_method_length - ] - idx += 1 + compression_method_length - # extension - if idx == length: - self.extension = b'' - else: - extension_length, = struct.unpack('!H', raw[idx: idx + 2]) - self.extension = raw[idx: idx + 2 + extension_length] - idx += 2 + extension_length - return True, raw[idx:] - except Exception as e: - logger.exception(e) - return False, raw - - def build(self) -> bytes: - return b''.join([ - bs for bs in ( - self.protocol_version, self.random, self.session_id, self.cipher_suite, - self.compression_method, self.extension, - ) if bs is not None - ]) - - def format(self) -> str: - parts = [] - parts.append( - 'Protocol Version: %s' % ( - pretty_hexlify(self.protocol_version) - if self.protocol_version is not None - else '' - ), - ) - parts.append( - 'Random: %s' % ( - pretty_hexlify(self.random) - if self.random is not None - else '' - ), - ) - parts.append( - 'Session ID: %s' % ( - pretty_hexlify(self.session_id) - if self.session_id is not None - else '' - ), - ) - parts.append( - 'Cipher Suite: %s' % ( - pretty_hexlify(self.cipher_suite) - if self.cipher_suite is not None - else '' - ), - ) - parts.append( - 'Compression Method: %s' % ( - pretty_hexlify(self.compression_method) - if self.compression_method is not None - else '' - ), - ) - parts.append( - 'Extension: %s' % ( - pretty_hexlify(self.extension) - if self.extension is not None - else '' - ), - ) - return os.linesep.join(parts) - - -class TlsCertificate: - """TLS Certificate""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - self.data = raw - return True, raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsServerKeyExchange: - """TLS Server Key Exchange""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - self.data = raw - return True, raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsCertificateRequest: - """TLS Certificate Request""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - return False, raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsServerHelloDone: - """TLS Server Hello Done""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - return False, raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsCertificateVerify: - """TLS Certificate Verify""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - return False, raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsClientKeyExchange: - """TLS Client Key Exchange""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - return False, raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsFinished: - """TLS Finished""" - - def __init__(self) -> None: - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - return False, raw - - def build(self) -> bytes: - assert self.data - return self.data - - -class TlsHandshake: - """TLS Handshake""" - - def __init__(self) -> None: - self.msg_type: int = tlsHandshakeType.OTHER - self.length: Optional[bytes] = None - self.hello_request: Optional[TlsHelloRequest] = None - self.client_hello: Optional[TlsClientHello] = None - self.server_hello: Optional[TlsServerHello] = None - self.certificate: Optional[TlsCertificate] = None - self.server_key_exchange: Optional[TlsServerKeyExchange] = None - self.certificate_request: Optional[TlsCertificateRequest] = None - self.server_hello_done: Optional[TlsServerHelloDone] = None - self.certificate_verify: Optional[TlsCertificateVerify] = None - self.client_key_exchange: Optional[TlsClientKeyExchange] = None - self.finished: Optional[TlsFinished] = None - self.data: Optional[bytes] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - length = len(raw) - if length < 4: - logger.debug('invalid data, len(raw) = %s', length) - return False, raw - else: - payload_length, = struct.unpack('!I', b'\x00' + raw[1:4]) - self.length = payload_length - if length < 4 + payload_length: - logger.debug( - 'incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length, - ) - return False, raw - # parse - self.msg_type = raw[0] - self.length = raw[1:4] - self.data = raw[: 4 + payload_length] - payload = raw[4: 4 + payload_length] - if self.msg_type == tlsHandshakeType.HELLO_REQUEST: - # parse hello request - self.hello_request = TlsHelloRequest() - self.hello_request.parse(payload) - elif self.msg_type == tlsHandshakeType.CLIENT_HELLO: - # parse client hello - self.client_hello = TlsClientHello() - self.client_hello.parse(payload) - elif self.msg_type == tlsHandshakeType.SERVER_HELLO: - # parse server hello - self.server_hello = TlsServerHello() - self.server_hello.parse(payload) - elif self.msg_type == tlsHandshakeType.CERTIFICATE: - # parse certificate - self.certificate = TlsCertificate() - self.certificate.parse(payload) - elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: - # parse server key exchange - self.server_key_exchange = TlsServerKeyExchange() - self.server_key_exchange.parse(payload) - elif self.msg_type == tlsHandshakeType.CERTIFICATE_REQUEST: - # parse certificate request - self.certificate_request = TlsCertificateRequest() - self.certificate_request.parse(payload) - elif self.msg_type == tlsHandshakeType.SERVER_HELLO_DONE: - # parse server hello done - self.server_hello_done = TlsServerHelloDone() - self.server_hello_done.parse(payload) - elif self.msg_type == tlsHandshakeType.CERTIFICATE_VERIFY: - # parse certificate verify - self.certificate_verify = TlsCertificateVerify() - self.certificate_verify.parse(payload) - elif self.msg_type == tlsHandshakeType.CLIENT_KEY_EXCHANGE: - # parse client key exchange - self.client_key_exchange = TlsClientKeyExchange() - self.client_key_exchange.parse(payload) - elif self.msg_type == tlsHandshakeType.FINISHED: - # parse finished - self.finished = TlsFinished() - self.finished.parse(payload) - return True, raw[4 + payload_length:] - - def build(self) -> bytes: - data = b'' - data += bytes([self.msg_type]) - payload = b'' - if self.msg_type == tlsHandshakeType.CLIENT_HELLO: - assert self.client_hello - payload = self.client_hello.build() - elif self.msg_type == tlsHandshakeType.SERVER_HELLO: - assert self.server_hello - payload = self.server_hello.build() - elif self.msg_type == tlsHandshakeType.CERTIFICATE: - assert self.certificate - payload = self.certificate.build() - elif self.msg_type == tlsHandshakeType.SERVER_KEY_EXCHANGE: - assert self.server_key_exchange - payload = self.server_key_exchange.build() - # calculate length - length = struct.pack('!I', len(payload))[1:] - data += length - data += payload - return data - - -class TlsParser: - """TLS packet parser""" - - def __init__(self) -> None: - self.content_type: int = tlsContentType.OTHER - self.protocol_version: Optional[bytes] = None - self.length: Optional[bytes] = None - # only parse hand shake payload temporary - self.handshake: Optional[TlsHandshake] = None - self.certificate: Optional[TlsCertificate] = None - - def parse(self, raw: bytes) -> Tuple[bool, bytes]: - """parse TLS fragmentation - Ref: https://datatracker.ietf.org/doc/html/rfc5246#page-15 - https://datatracker.ietf.org/doc/html/rfc5077#page-3 - https://datatracker.ietf.org/doc/html/rfc8446#page-10 - """ - length = len(raw) - if length < 5: - logger.debug('invalid data, len(raw) = %s', length) - return False, raw - else: - payload_length, = struct.unpack('!H', raw[3:5]) - if length < 5 + payload_length: - logger.debug( - 'incomplete data, len(raw) = %s, len(payload) = %s', length, payload_length, - ) - return False, raw - else: - # parse - self.content_type = raw[0] - # ??? - self.protocol_version = raw[1:3] - self.length = raw[3:5] - payload = raw[5:5 + payload_length] - if self.content_type == tlsContentType.HANDSHAKE: - # parse handshake - self.handshake = TlsHandshake() - self.handshake.parse(payload) - return True, raw[5 + payload_length:] - - def build(self) -> bytes: - data = b'' - data += bytes([self.content_type]) - # ??? - data += self.protocol_version - payload = b'' - if self.content_type == tlsContentType.HANDSHAKE: - assert self.handshake - payload += self.handshake.build() - length = struct.pack('!H', len(payload)) - data += length - data += payload - return data diff --git a/tests/http/test_tls_parser.py b/tests/http/test_tls_parser.py deleted file mode 100644 index 79dd7a6f76..0000000000 --- a/tests/http/test_tls_parser.py +++ /dev/null @@ -1,69 +0,0 @@ -# -*- coding: utf-8 -*- -""" - proxy.py - ~~~~~~~~ - ⚡⚡⚡ Fast, Lightweight, Pluggable, TLS interception capable proxy server focused on - Network monitoring, controls & Application development, testing, debugging. - - :copyright: (c) 2013-present by Abhinav Singh and contributors. - :license: BSD, see LICENSE for more details. -""" -import unittest -import binascii -import re -import os - -from proxy.http.parser.tls import TlsParser -from proxy.http.parser.tls import tlsContentType, tlsHandshakeType - - -class TestTlsParser(unittest.TestCase): - """Ref: https://tls.ulfheim.net/""" - - def unhexlify(self, raw: str) -> bytes: - return binascii.unhexlify(re.sub(r'\s', '', raw)) - - def test_parse_client_hello(self) -> None: - data = """ - 16 03 01 00 a5 01 00 00 a1 03 03 00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 - 15 16 17 18 19 1a 1b 1c 1d 1e 1f 00 00 20 cc a8 cc a9 c0 2f c0 30 c0 2b c0 2c c0 13 c0 09 c0 14 - c0 0a 00 9c 00 9d 00 2f 00 35 c0 12 00 0a 01 00 00 58 00 00 00 18 00 16 00 00 13 65 78 61 6d 70 - 6c 65 2e 75 6c 66 68 65 69 6d 2e 6e 65 74 00 05 00 05 01 00 00 00 00 00 0a 00 0a 00 08 00 1d 00 - 17 00 18 00 19 00 0b 00 02 01 00 00 0d 00 12 00 10 04 01 04 03 05 01 05 03 06 01 06 03 02 01 02 - 03 ff 01 00 01 00 00 12 00 00 - """ - tls = TlsParser() - tls.parse(self.unhexlify(data)) - self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE) - self.assertEqual(tls.length, b'\x00\xa5') - assert tls.handshake - self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.CLIENT_HELLO) - self.assertEqual(tls.handshake.length, b'\x00\x00\xa1') - self.assertEqual(len(tls.handshake.build()), 0xA1 + 0x04) - assert tls.handshake.client_hello - self.assertEqual( - tls.handshake.client_hello.protocol_version, b'\x03\x03', - ) - self.assertEqual( - tls.handshake.client_hello.random, self.unhexlify( - '00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f 10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f', - ), - ) - - def test_parse_server_hello(self) -> None: - data = open( - os.path.join( - os.path.dirname(__file__), - 'tls_server_hello.data', - ), 'r', - ).read() - tls = TlsParser() - tls.parse(self.unhexlify(data)) - self.assertEqual(tls.content_type, tlsContentType.HANDSHAKE) - assert tls.handshake - self.assertEqual(tls.handshake.msg_type, tlsHandshakeType.SERVER_HELLO) - assert tls.handshake.server_hello - self.assertEqual( - tls.handshake.server_hello.protocol_version, b'\x03\x03', - ) - print(tls.handshake.server_hello.format()) diff --git a/tests/http/tls_server_hello.data b/tests/http/tls_server_hello.data deleted file mode 100644 index 72553c3153..0000000000 --- a/tests/http/tls_server_hello.data +++ /dev/null @@ -1,337 +0,0 @@ -16 03 03 00 59 02 00 00 55 03 03 60 f9 54 e7 1b -c2 60 2d 02 94 ed 2a 82 d2 1d 06 71 3c 52 d1 23 -0e 73 fe c9 93 8a 05 71 f5 48 46 20 b0 d9 d0 1c -bd 6b 49 27 47 88 14 c6 ef 04 bc 48 3b 01 5f 00 -61 ae f8 a2 45 5e 7b fe fb ed f4 a8 c0 2f 00 00 -0d ff 01 00 01 00 00 0b 00 04 03 00 01 02 16 03 -03 13 4f 0b 00 13 4b 00 13 48 00 05 e0 30 82 05 -dc 30 82 04 c4 a0 03 02 01 02 02 10 01 58 f3 0a -d5 9d 62 af 20 9f 15 ee 4f c2 bb 33 30 0d 06 09 -2a 86 48 86 f7 0d 01 01 0b 05 00 30 46 31 0b 30 -09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06 03 -55 04 0a 13 06 41 6d 61 7a 6f 6e 31 15 30 13 06 -03 55 04 0b 13 0c 53 65 72 76 65 72 20 43 41 20 -31 42 31 0f 30 0d 06 03 55 04 03 13 06 41 6d 61 -7a 6f 6e 30 1e 17 0d 32 31 31 31 32 31 30 30 30 -30 30 30 5a 17 0d 32 32 31 32 31 39 32 33 35 39 -35 39 5a 30 16 31 14 30 12 06 03 55 04 03 13 0b -68 74 74 70 62 69 6e 2e 6f 72 67 30 82 01 22 30 -0d 06 09 2a 86 48 86 f7 0d 01 01 01 05 00 03 82 -01 0f 00 30 82 01 0a 02 82 01 01 00 84 e4 27 a5 -ec eb c0 0d 2f 1f 37 f8 ec f6 be 3b ce 1f 5a e7 -bf e7 ad 93 a9 0a d5 8a bb 5f fb 77 ec 19 07 77 -32 6c 27 21 df a4 b0 01 90 bd 63 78 33 5f e1 49 -e9 58 25 bd b8 ea 91 6a 18 1e ed f9 92 3e be 30 -58 1f 8f c9 e8 28 85 b0 ac 33 ba ca a4 48 5e b7 -7e d4 21 f0 ec 74 39 ab 01 f4 7b d1 94 e3 c7 4f -f0 79 0e c9 21 f0 73 62 27 ed 65 13 14 99 11 a3 -bd ce c4 3d c0 ee b6 6e c1 87 b0 c5 ff 50 fd d5 -40 1c 97 2d 50 59 bb 40 cc 2a c9 76 84 66 94 37 -2f 7e 2e 06 6f e3 f8 95 a5 60 cf 43 5f a4 86 97 -f1 3f dc 75 1f e9 8c 95 45 81 81 83 ea ee 31 b9 -71 e4 db 96 d2 28 17 fb 14 40 f3 51 fb 22 88 05 -29 ee 88 84 97 14 73 bd 1c 49 e4 e6 98 7a d7 48 -56 99 54 0a 22 b0 80 3e c0 14 30 cf 66 eb e8 83 -6f ae 0d 8e b3 23 59 5f db 92 6d fa af e1 41 33 -3f 4b f0 77 c4 07 88 71 97 0c 20 e5 02 03 01 00 -01 a3 82 02 f4 30 82 02 f0 30 1f 06 03 55 1d 23 -04 18 30 16 80 14 59 a4 66 06 52 a0 7b 95 92 3c -a3 94 07 27 96 74 5b f9 3d d0 30 1d 06 03 55 1d -0e 04 16 04 14 cf 07 84 b8 03 48 ac c9 ac db 11 -65 0a 7d 29 ff d6 97 4b b3 30 25 06 03 55 1d 11 -04 1e 30 1c 82 0b 68 74 74 70 62 69 6e 2e 6f 72 -67 82 0d 2a 2e 68 74 74 70 62 69 6e 2e 6f 72 67 -30 0e 06 03 55 1d 0f 01 01 ff 04 04 03 02 05 a0 -30 1d 06 03 55 1d 25 04 16 30 14 06 08 2b 06 01 -05 05 07 03 01 06 08 2b 06 01 05 05 07 03 02 30 -3d 06 03 55 1d 1f 04 36 30 34 30 32 a0 30 a0 2e -86 2c 68 74 74 70 3a 2f 2f 63 72 6c 2e 73 63 61 -31 62 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63 -6f 6d 2f 73 63 61 31 62 2d 31 2e 63 72 6c 30 13 -06 03 55 1d 20 04 0c 30 0a 30 08 06 06 67 81 0c -01 02 01 30 75 06 08 2b 06 01 05 05 07 01 01 04 -69 30 67 30 2d 06 08 2b 06 01 05 05 07 30 01 86 -21 68 74 74 70 3a 2f 2f 6f 63 73 70 2e 73 63 61 -31 62 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63 -6f 6d 30 36 06 08 2b 06 01 05 05 07 30 02 86 2a -68 74 74 70 3a 2f 2f 63 72 74 2e 73 63 61 31 62 -2e 61 6d 61 7a 6f 6e 74 72 75 73 74 2e 63 6f 6d -2f 73 63 61 31 62 2e 63 72 74 30 0c 06 03 55 1d -13 01 01 ff 04 02 30 00 30 82 01 7d 06 0a 2b 06 -01 04 01 d6 79 02 04 02 04 82 01 6d 04 82 01 69 -01 67 00 76 00 29 79 be f0 9e 39 39 21 f0 56 73 -9f 63 a5 77 e5 be 57 7d 9c 60 0a f8 f9 4d 5d 26 -5c 25 5d c7 84 00 00 01 7d 40 73 08 b5 00 00 04 -03 00 47 30 45 02 20 5d d8 f1 cc 14 99 9f 5c 99 -84 9c 94 da d2 60 e8 58 0d e0 b3 49 30 18 1f 0a -48 86 9e a8 00 72 81 02 21 00 ff c9 ca c0 dd 06 -dd 2c 14 89 2d ff 35 dc e5 56 eb 9a 92 be 1a ee -8b e2 03 33 7b cd 78 92 9b 37 00 76 00 51 a3 b0 -f5 fd 01 79 9c 56 6d b8 37 78 8f 0c a4 7a cc 1b -27 cb f7 9e 88 42 9a 0d fe d4 8b 05 e5 00 00 01 -7d 40 73 08 c9 00 00 04 03 00 47 30 45 02 20 7e -29 67 be 79 d6 34 0e 27 f9 55 db 4b 03 28 d0 96 -38 b7 6d f9 c4 b6 a6 35 08 14 4a 48 f2 7c 25 02 -21 00 8f 68 71 a6 2b c8 3e d5 fd b5 17 84 70 90 -9a 02 e0 b9 1a 1f 74 99 9c fd 95 64 32 ba c2 03 -d1 54 00 75 00 df a5 5e ab 68 82 4f 1f 6c ad ee -b8 5f 4e 3e 5a ea cd a2 12 a4 6a 5e 8e 3b 12 c0 -20 44 5c 2a 73 00 00 01 7d 40 73 08 a2 00 00 04 -03 00 46 30 44 02 20 4c 9e 68 c3 ac cb 92 d2 b0 -a9 08 a1 dd 82 7c 13 0b dc 9f fe 0b 08 ef 25 e9 -1d 30 12 38 41 82 36 02 20 18 29 da 43 a6 d0 df -b5 71 bb 19 23 b3 f1 28 29 a9 1b 54 a1 62 07 b3 -5c 28 4a 3e ee c0 42 70 bb 30 0d 06 09 2a 86 48 -86 f7 0d 01 01 0b 05 00 03 82 01 01 00 2f 64 1d -d3 7d 3c 06 41 7b 6a 1c 94 60 99 31 92 b7 eb e1 -6c b2 ac ee d2 5b f4 ec 36 94 a6 c6 a3 c5 f8 3f -d0 a9 07 01 b7 cb 4f 1d 68 35 a0 d1 1c 51 88 a7 -97 9a ad 03 89 32 1e aa 40 96 0c d8 97 4e 05 52 -94 a1 90 d0 e3 73 07 75 1a 29 20 98 d3 08 85 21 -b1 dc 41 da 41 ec 87 be b2 88 80 24 5e 4c ad 5f -3c fc 43 bc 55 36 6a 8f 67 9c 47 e2 43 43 a8 b2 -c4 55 0b a1 1a 26 c4 3e fc 21 27 4e 2a f4 04 43 -e3 59 18 d6 d8 2f e0 f5 83 87 47 23 fd 3c ee 11 -ff 9f 0a 3b 2f 29 74 96 70 5a 43 b2 18 ac a5 65 -e9 ac 2f 2e ac 47 c8 77 50 4d 0a 5b 44 da c0 cc -06 3b eb 23 59 4d 9e 07 fb 8e 27 a6 1c 68 04 b4 -a2 0d 52 ec da cb 0e a4 f7 1d 45 92 7f cd 60 ee -a9 4d 78 2d e1 36 02 3c 7c cc 45 2c f4 e4 86 7f -3e bf 6d b4 48 98 a9 8b a8 a0 9e 42 51 54 5a 02 -d5 a4 c5 a2 55 cc ab fa 0c a9 1a 94 1c 00 04 4d -30 82 04 49 30 82 03 31 a0 03 02 01 02 02 13 06 -7f 94 57 85 87 e8 ac 77 de b2 53 32 5b bc 99 8b -56 0d 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 -00 30 39 31 0b 30 09 06 03 55 04 06 13 02 55 53 -31 0f 30 0d 06 03 55 04 0a 13 06 41 6d 61 7a 6f -6e 31 19 30 17 06 03 55 04 03 13 10 41 6d 61 7a -6f 6e 20 52 6f 6f 74 20 43 41 20 31 30 1e 17 0d -31 35 31 30 32 32 30 30 30 30 30 30 5a 17 0d 32 -35 31 30 31 39 30 30 30 30 30 30 5a 30 46 31 0b -30 09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06 -03 55 04 0a 13 06 41 6d 61 7a 6f 6e 31 15 30 13 -06 03 55 04 0b 13 0c 53 65 72 76 65 72 20 43 41 -20 31 42 31 0f 30 0d 06 03 55 04 03 13 06 41 6d -61 7a 6f 6e 30 82 01 22 30 0d 06 09 2a 86 48 86 -f7 0d 01 01 01 05 00 03 82 01 0f 00 30 82 01 0a -02 82 01 01 00 c2 4e 16 67 dd ce bc 6a c8 37 5a -ec 3a 30 b0 1d e6 d1 12 e8 12 28 48 cc e8 29 c1 -b9 6e 53 d5 a3 eb 03 39 1a cc 77 87 f6 01 b9 d9 -70 cc cf 6b 8d e3 e3 03 71 86 99 6d cb a6 94 2a -4e 13 d6 a7 bd 04 ec 0a 16 3c 0a eb 39 b1 c4 b5 -58 a3 b6 c7 56 25 ec 3e 52 7a a8 e3 29 16 07 b9 -6e 50 cf fb 5f 31 f8 1d ba 03 4a 62 89 03 ae 3e -47 f2 0f 27 91 e3 14 20 85 f8 fa e9 8a 35 f5 5f -9e 99 4d e7 6b 37 ef a4 50 3e 44 ec fa 5a 85 66 -07 9c 7e 17 6a 55 f3 17 8a 35 1e ee e9 ac c3 75 -4e 58 55 7d 53 6b 0a 6b 9b 14 42 d7 e5 ac 01 89 -b3 ea a3 fe cf c0 2b 0c 84 c2 d8 53 15 cb 67 f0 -d0 88 ca 3a d1 17 73 f5 5f 9a d4 c5 72 1e 7e 01 -f1 98 30 63 2a aa f2 7a 2d c5 e2 02 1a 86 e5 32 -3e 0e bd 11 b4 cf 3c 93 ef 17 50 10 9e 43 c2 06 -2a e0 0d 68 be d3 88 8b 4a 65 8c 4a d4 c3 2e 4c -9b 55 f4 86 e5 02 03 01 00 01 a3 82 01 3b 30 82 -01 37 30 12 06 03 55 1d 13 01 01 ff 04 08 30 06 -01 01 ff 02 01 00 30 0e 06 03 55 1d 0f 01 01 ff -04 04 03 02 01 86 30 1d 06 03 55 1d 0e 04 16 04 -14 59 a4 66 06 52 a0 7b 95 92 3c a3 94 07 27 96 -74 5b f9 3d d0 30 1f 06 03 55 1d 23 04 18 30 16 -80 14 84 18 cc 85 34 ec bc 0c 94 94 2e 08 59 9c -c7 b2 10 4e 0a 08 30 7b 06 08 2b 06 01 05 05 07 -01 01 04 6f 30 6d 30 2f 06 08 2b 06 01 05 05 07 -30 01 86 23 68 74 74 70 3a 2f 2f 6f 63 73 70 2e -72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 72 -75 73 74 2e 63 6f 6d 30 3a 06 08 2b 06 01 05 05 -07 30 02 86 2e 68 74 74 70 3a 2f 2f 63 72 74 2e -72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 72 -75 73 74 2e 63 6f 6d 2f 72 6f 6f 74 63 61 31 2e -63 65 72 30 3f 06 03 55 1d 1f 04 38 30 36 30 34 -a0 32 a0 30 86 2e 68 74 74 70 3a 2f 2f 63 72 6c -2e 72 6f 6f 74 63 61 31 2e 61 6d 61 7a 6f 6e 74 -72 75 73 74 2e 63 6f 6d 2f 72 6f 6f 74 63 61 31 -2e 63 72 6c 30 13 06 03 55 1d 20 04 0c 30 0a 30 -08 06 06 67 81 0c 01 02 01 30 0d 06 09 2a 86 48 -86 f7 0d 01 01 0b 05 00 03 82 01 01 00 85 92 be -35 bb 79 cf a3 81 42 1c e4 e3 63 73 53 39 52 35 -e7 d1 ad fd ae 99 8a ac 89 12 2f bb e7 6f 9a d5 -4e 72 ea 20 30 61 f9 97 b2 cd a5 27 02 45 a8 ca -76 3e 98 4a 83 9e b6 e6 45 e0 f2 43 f6 08 de 6d -e8 6e db 31 07 13 f0 2f 31 0d 93 6d 61 37 7b 58 -f0 fc 51 98 91 28 02 4f 05 76 b7 d3 f0 1b c2 e6 -5e d0 66 85 11 0f 2e 81 c6 10 81 29 fe 20 60 48 -f3 f2 f0 84 13 53 65 35 15 11 6b 82 51 40 55 57 -5f 18 b5 b0 22 3e ad f2 5e a3 01 e3 c3 b3 f9 cb -41 5a e6 52 91 bb e4 36 87 4f 2d a9 a4 07 68 35 -ba 94 72 cd 0e ea 0e 7d 57 f2 79 fc 37 c5 7b 60 -9e b2 eb c0 2d 90 77 0d 49 10 27 a5 38 ad c4 12 -a3 b4 a3 c8 48 b3 15 0b 1e e2 e2 19 dc c4 76 52 -c8 bc 8a 41 78 70 d9 6d 97 b3 4a 8b 78 2d 5e b4 -0f a3 4c 60 ca e1 47 cb 78 2d 12 17 b1 52 8b ca -39 2c bd b5 2f c2 33 02 96 ab da 94 7f 00 04 96 -30 82 04 92 30 82 03 7a a0 03 02 01 02 02 13 06 -7f 94 4a 2a 27 cd f3 fa c2 ae 2b 01 f9 08 ee b9 -c4 c6 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 -00 30 81 98 31 0b 30 09 06 03 55 04 06 13 02 55 -53 31 10 30 0e 06 03 55 04 08 13 07 41 72 69 7a -6f 6e 61 31 13 30 11 06 03 55 04 07 13 0a 53 63 -6f 74 74 73 64 61 6c 65 31 25 30 23 06 03 55 04 -0a 13 1c 53 74 61 72 66 69 65 6c 64 20 54 65 63 -68 6e 6f 6c 6f 67 69 65 73 2c 20 49 6e 63 2e 31 -3b 30 39 06 03 55 04 03 13 32 53 74 61 72 66 69 -65 6c 64 20 53 65 72 76 69 63 65 73 20 52 6f 6f -74 20 43 65 72 74 69 66 69 63 61 74 65 20 41 75 -74 68 6f 72 69 74 79 20 2d 20 47 32 30 1e 17 0d -31 35 30 35 32 35 31 32 30 30 30 30 5a 17 0d 33 -37 31 32 33 31 30 31 30 30 30 30 5a 30 39 31 0b -30 09 06 03 55 04 06 13 02 55 53 31 0f 30 0d 06 -03 55 04 0a 13 06 41 6d 61 7a 6f 6e 31 19 30 17 -06 03 55 04 03 13 10 41 6d 61 7a 6f 6e 20 52 6f -6f 74 20 43 41 20 31 30 82 01 22 30 0d 06 09 2a -86 48 86 f7 0d 01 01 01 05 00 03 82 01 0f 00 30 -82 01 0a 02 82 01 01 00 b2 78 80 71 ca 78 d5 e3 -71 af 47 80 50 74 7d 6e d8 d7 88 76 f4 99 68 f7 -58 21 60 f9 74 84 01 2f ac 02 2d 86 d3 a0 43 7a -4e b2 a4 d0 36 ba 01 be 8d db 48 c8 07 17 36 4c -f4 ee 88 23 c7 3e eb 37 f5 b5 19 f8 49 68 b0 de -d7 b9 76 38 1d 61 9e a4 fe 82 36 a5 e5 4a 56 e4 -45 e1 f9 fd b4 16 fa 74 da 9c 9b 35 39 2f fa b0 -20 50 06 6c 7a d0 80 b2 a6 f9 af ec 47 19 8f 50 -38 07 dc a2 87 39 58 f8 ba d5 a9 f9 48 67 30 96 -ee 94 78 5e 6f 89 a3 51 c0 30 86 66 a1 45 66 ba -54 eb a3 c3 91 f9 48 dc ff d1 e8 30 2d 7d 2d 74 -70 35 d7 88 24 f7 9e c4 59 6e bb 73 87 17 f2 32 -46 28 b8 43 fa b7 1d aa ca b4 f2 9f 24 0e 2d 4b -f7 71 5c 5e 69 ff ea 95 02 cb 38 8a ae 50 38 6f -db fb 2d 62 1b c5 c7 1e 54 e1 77 e0 67 c8 0f 9c -87 23 d6 3f 40 20 7f 20 80 c4 80 4c 3e 3b 24 26 -8e 04 ae 6c 9a c8 aa 0d 02 03 01 00 01 a3 82 01 -31 30 82 01 2d 30 0f 06 03 55 1d 13 01 01 ff 04 -05 30 03 01 01 ff 30 0e 06 03 55 1d 0f 01 01 ff -04 04 03 02 01 86 30 1d 06 03 55 1d 0e 04 16 04 -14 84 18 cc 85 34 ec bc 0c 94 94 2e 08 59 9c c7 -b2 10 4e 0a 08 30 1f 06 03 55 1d 23 04 18 30 16 -80 14 9c 5f 00 df aa 01 d7 30 2b 38 88 a2 b8 6d -4a 9c f2 11 91 83 30 78 06 08 2b 06 01 05 05 07 -01 01 04 6c 30 6a 30 2e 06 08 2b 06 01 05 05 07 -30 01 86 22 68 74 74 70 3a 2f 2f 6f 63 73 70 2e -72 6f 6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 -73 74 2e 63 6f 6d 30 38 06 08 2b 06 01 05 05 07 -30 02 86 2c 68 74 74 70 3a 2f 2f 63 72 74 2e 72 -6f 6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 73 -74 2e 63 6f 6d 2f 72 6f 6f 74 67 32 2e 63 65 72 -30 3d 06 03 55 1d 1f 04 36 30 34 30 32 a0 30 a0 -2e 86 2c 68 74 74 70 3a 2f 2f 63 72 6c 2e 72 6f -6f 74 67 32 2e 61 6d 61 7a 6f 6e 74 72 75 73 74 -2e 63 6f 6d 2f 72 6f 6f 74 67 32 2e 63 72 6c 30 -11 06 03 55 1d 20 04 0a 30 08 30 06 06 04 55 1d -20 00 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 -00 03 82 01 01 00 62 37 42 5c bc 10 b5 3e 8b 2c -e9 0c 9b 6c 45 e2 07 00 7a f9 c5 58 0b b9 08 8c -3e ed b3 25 3c b5 6f 50 e4 cd 35 6a a7 93 34 96 -32 21 a9 48 44 ab 9c ed 3d b4 aa 73 6d e4 7f 16 -80 89 6c cf 28 03 18 83 47 79 a3 10 7e 30 5b ac -3b b0 60 e0 77 d4 08 a6 e1 1d 7c 5e c0 bb f9 9a -7b 22 9d a7 00 09 7e ac 46 17 83 dc 9c 26 57 99 -30 39 62 96 8f ed da de aa c5 cc 1b 3e ca 43 68 -6c 57 16 bc d5 0e 20 2e fe ff c2 6a 5d 2e a0 4a -6d 14 58 87 94 e6 39 31 5f 7c 73 cb 90 88 6a 84 -11 96 27 a6 ed d9 81 46 a6 7e a3 72 00 0a 52 3e -83 88 07 63 77 89 69 17 0f 39 85 d2 ab 08 45 4d -d0 51 3a fd 5d 5d 37 64 4c 7e 30 b2 55 24 42 9d -36 b0 5d 9c 17 81 61 f1 ca f9 10 02 24 ab eb 0d -74 91 8d 7b 45 29 50 39 88 b2 a6 89 35 25 1e 14 -6a 47 23 31 2f 5c 9a fa ad 9a 0e 62 51 a4 2a a9 -c4 f9 34 9d 21 18 00 04 79 30 82 04 75 30 82 03 -5d a0 03 02 01 02 02 09 00 a7 0e 4a 4c 34 82 b7 -7f 30 0d 06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 -30 68 31 0b 30 09 06 03 55 04 06 13 02 55 53 31 -25 30 23 06 03 55 04 0a 13 1c 53 74 61 72 66 69 -65 6c 64 20 54 65 63 68 6e 6f 6c 6f 67 69 65 73 -2c 20 49 6e 63 2e 31 32 30 30 06 03 55 04 0b 13 -29 53 74 61 72 66 69 65 6c 64 20 43 6c 61 73 73 -20 32 20 43 65 72 74 69 66 69 63 61 74 69 6f 6e -20 41 75 74 68 6f 72 69 74 79 30 1e 17 0d 30 39 -30 39 30 32 30 30 30 30 30 30 5a 17 0d 33 34 30 -36 32 38 31 37 33 39 31 36 5a 30 81 98 31 0b 30 -09 06 03 55 04 06 13 02 55 53 31 10 30 0e 06 03 -55 04 08 13 07 41 72 69 7a 6f 6e 61 31 13 30 11 -06 03 55 04 07 13 0a 53 63 6f 74 74 73 64 61 6c -65 31 25 30 23 06 03 55 04 0a 13 1c 53 74 61 72 -66 69 65 6c 64 20 54 65 63 68 6e 6f 6c 6f 67 69 -65 73 2c 20 49 6e 63 2e 31 3b 30 39 06 03 55 04 -03 13 32 53 74 61 72 66 69 65 6c 64 20 53 65 72 -76 69 63 65 73 20 52 6f 6f 74 20 43 65 72 74 69 -66 69 63 61 74 65 20 41 75 74 68 6f 72 69 74 79 -20 2d 20 47 32 30 82 01 22 30 0d 06 09 2a 86 48 -86 f7 0d 01 01 01 05 00 03 82 01 0f 00 30 82 01 -0a 02 82 01 01 00 d5 0c 3a c4 2a f9 4e e2 f5 be -19 97 5f 8e 88 53 b1 1f 3f cb cf 9f 20 13 6d 29 -3a c8 0f 7d 3c f7 6b 76 38 63 d9 36 60 a8 9b 5e -5c 00 80 b2 2f 59 7f f6 87 f9 25 43 86 e7 69 1b -52 9a 90 e1 71 e3 d8 2d 0d 4e 6f f6 c8 49 d9 b6 -f3 1a 56 ae 2b b6 74 14 eb cf fb 26 e3 1a ba 1d -96 2e 6a 3b 58 94 89 47 56 ff 25 a0 93 70 53 83 -da 84 74 14 c3 67 9e 04 68 3a df 8e 40 5a 1d 4a -4e cf 43 91 3b e7 56 d6 00 70 cb 52 ee 7b 7d ae -3a e7 bc 31 f9 45 f6 c2 60 cf 13 59 02 2b 80 cc -34 47 df b9 de 90 65 6d 02 cf 2c 91 a6 a6 e7 de -85 18 49 7c 66 4e a3 3a 6d a9 b5 ee 34 2e ba 0d -03 b8 33 df 47 eb b1 6b 8d 25 d9 9b ce 81 d1 45 -46 32 96 70 87 de 02 0e 49 43 85 b6 6c 73 bb 64 -ea 61 41 ac c9 d4 54 df 87 2f c7 22 b2 26 cc 9f -59 54 68 9f fc be 2a 2f c4 55 1c 75 40 60 17 85 -02 55 39 8b 7f 05 02 03 01 00 01 a3 81 f0 30 81 -ed 30 0f 06 03 55 1d 13 01 01 ff 04 05 30 03 01 -01 ff 30 0e 06 03 55 1d 0f 01 01 ff 04 04 03 02 -01 86 30 1d 06 03 55 1d 0e 04 16 04 14 9c 5f 00 -df aa 01 d7 30 2b 38 88 a2 b8 6d 4a 9c f2 11 91 -83 30 1f 06 03 55 1d 23 04 18 30 16 80 14 bf 5f -b7 d1 ce dd 1f 86 f4 5b 55 ac dc d7 10 c2 0e a9 -88 e7 30 4f 06 08 2b 06 01 05 05 07 01 01 04 43 -30 41 30 1c 06 08 2b 06 01 05 05 07 30 01 86 10 -68 74 74 70 3a 2f 2f 6f 2e 73 73 32 2e 75 73 2f -30 21 06 08 2b 06 01 05 05 07 30 02 86 15 68 74 -74 70 3a 2f 2f 78 2e 73 73 32 2e 75 73 2f 78 2e -63 65 72 30 26 06 03 55 1d 1f 04 1f 30 1d 30 1b -a0 19 a0 17 86 15 68 74 74 70 3a 2f 2f 73 2e 73 -73 32 2e 75 73 2f 72 2e 63 72 6c 30 11 06 03 55 -1d 20 04 0a 30 08 30 06 06 04 55 1d 20 00 30 0d -06 09 2a 86 48 86 f7 0d 01 01 0b 05 00 03 82 01 -01 00 23 1d e3 8a 57 ca 7d e9 17 79 4c f1 1e 55 -fd cc 53 6e 3e 47 0f df c6 55 f2 b2 04 36 ed 80 -1f 53 c4 5d 34 28 6b be c7 55 fc 67 ea cb 3f 7f -90 b2 33 cd 1b 58 10 82 02 f8 f8 2f f5 13 60 d4 -05 ce f1 81 08 c1 dd a7 75 97 4f 18 b9 6d de f7 -93 91 08 ba 7e 40 2c ed c1 ea bb 76 9e 33 06 77 -1d 0d 08 7f 53 dd 1b 64 ab 82 27 f1 69 d5 4d 5e -ae f4 a1 c3 75 a7 58 44 2d f2 3c 70 98 ac ba 69 -b6 95 77 7f 0f 31 5e 2c fc a0 87 3a 47 69 f0 79 -5f f4 14 54 a4 95 5e 11 78 12 60 27 ce 9f c2 77 -ff 23 53 77 5d ba ff ea 59 e7 db cf af 92 96 ef -24 9a 35 10 7a 9c 91 c6 0e 7d 99 f6 3f 19 df f5 -72 54 e1 15 a9 07 59 7b 83 bf 52 2e 46 8c b2 00 -64 76 1c 48 d3 d8 79 e8 6e 56 cc ae 2c 03 90 d7 -19 38 99 e4 ca 09 19 5b ff 07 96 b0 a8 7f 34 49 -df 56 a9 f7 b0 5f ed 33 ed 8c 47 b7 30 03 5d f4 -03 8c 16 03 03 01 4d 0c 00 01 49 03 00 17 41 04 -e4 b3 e7 a8 fe ec e3 99 76 95 8d a5 32 ce de bd -ce 8c 2b 3d 31 f4 97 88 9c 39 e5 a0 1f 37 ec f9 -3d b1 e8 5c 5e a2 da 02 6d 91 e5 41 51 ca 72 6e -68 61 77 c8 3a e6 c1 d2 84 c7 4b 64 de 05 06 70 -06 01 01 00 28 3b e1 f9 23 fd 39 af 3a fa d6 f2 -ad e9 92 ae 66 49 90 58 36 19 2d 21 7b 0b 71 48 -ef 61 3f fe a0 db 3c 61 70 7f 3f cf 14 b5 a9 a8 -5b fa 45 45 8c f2 3f 13 5e da 19 f4 91 dd 2d f1 -8a 29 f6 7a 04 66 d7 54 14 38 b6 19 9d 07 66 a0 -1d c2 a5 cc ac a4 50 f1 d0 71 7a 75 84 a5 87 bd -7b f6 3d 75 6f 66 0f 28 0f 72 17 15 1b a4 fc 16 -9f 97 03 17 3a 88 a9 62 ce bc f3 cf 16 e1 e6 70 -d3 ad 65 dc fb 25 55 88 40 bf 5f b7 35 ac 4b 48 -9e f8 6b ab 6e b7 4d 10 41 a9 c8 19 d2 d7 76 51 -c9 d6 49 92 10 7e 7a 52 d8 ba 25 44 d3 4a 59 56 -15 49 aa 12 b0 ea 56 dc cf b1 43 07 ca cd 61 49 -e3 46 19 a8 1b 24 34 e5 c8 5f a5 cd 81 08 de b6 -6e 19 85 19 fc f0 d5 d1 98 19 1d 1a 09 fc d2 36 -87 c0 19 70 fc ca 1d 37 dc bd 1d 33 c3 8d 6f 2d -5d 29 4a ec 7f f8 f5 cf ea c8 de 47 8f 6f 17 42 -c4 23 4f 76 16 03 03 00 04 0e 00 00 00 \ No newline at end of file From 82d8d5c5a9f9c608819ac9b57ee46f2149173b66 Mon Sep 17 00:00:00 2001 From: Jerry Date: Tue, 21 Dec 2021 11:50:44 +0800 Subject: [PATCH 12/14] fix spell check warnings fix spell check warnings --- proxy/core/connection/pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 43ba407274..1363ca6d96 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -92,7 +92,7 @@ def release(self, conn: TcpServerConnection) -> None: """Release the connection. If the connection has not been closed, - then it will be retained in the pool for reusability. + then it will be retained in the pool for re-usability. """ if conn.closed: logger.debug( @@ -108,5 +108,5 @@ def release(self, conn: TcpServerConnection) -> None: ), ) assert not conn.is_reusable() - # Reset for reusability + # Reset for re-usability conn.reset() From 64190268f2119a71f6b42086f9ee416d0eb1ace8 Mon Sep 17 00:00:00 2001 From: Jerry Date: Wed, 22 Dec 2021 11:46:13 +0800 Subject: [PATCH 13/14] remove unnecessary statements remove unnecessary statements --- proxy/core/connection/connection.py | 2 -- proxy/core/connection/pool.py | 1 - proxy/core/connection/server.py | 2 -- proxy/http/proxy/server.py | 1 - 4 files changed, 6 deletions(-) diff --git a/proxy/core/connection/connection.py b/proxy/core/connection/connection.py index 4d460d7d99..84c6c9de74 100644 --- a/proxy/core/connection/connection.py +++ b/proxy/core/connection/connection.py @@ -52,7 +52,6 @@ def connection(self) -> Union[ssl.SSLSocket, socket.socket]: def send(self, data: bytes) -> int: """Users must handle BrokenPipeError exceptions""" # logger.info(data) - logger.info('send to %s, data = |%s|', self.tag, data) return self.connection.send(data) def recv( @@ -67,7 +66,6 @@ def recv( (len(data), self.tag), ) # logger.info(data) - logger.info('recv from %s, data = |%s|', self.tag, data) return memoryview(data) def close(self) -> bool: diff --git a/proxy/core/connection/pool.py b/proxy/core/connection/pool.py index 1363ca6d96..67413cc781 100644 --- a/proxy/core/connection/pool.py +++ b/proxy/core/connection/pool.py @@ -60,7 +60,6 @@ class ConnectionPool: def __init__(self) -> None: # Pools of connection per upstream server self.pools: Dict[Tuple[str, int], Set[TcpServerConnection]] = {} - self.handshake_pools: Dict[Tuple[str, int], Dict[int, bytes]] = {} def acquire(self, host: str, port: int) -> Tuple[bool, TcpServerConnection]: """Returns a connection for use with the server.""" diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index 7ab09dc904..fb201ada1d 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -27,8 +27,6 @@ def __init__(self, host: str, port: int) -> None: self._conn: Optional[Union[ssl.SSLSocket, socket.socket]] = None self.addr: Tuple[str, int] = (host, int(port)) self.closed = True - self.handshake: Dict[int, bytes] = {} - self.session_id: Optional[bytes] = None @property def connection(self) -> Union[ssl.SSLSocket, socket.socket]: diff --git a/proxy/http/proxy/server.py b/proxy/http/proxy/server.py index 5975cf9196..48e39c0967 100644 --- a/proxy/http/proxy/server.py +++ b/proxy/http/proxy/server.py @@ -31,7 +31,6 @@ from ..plugin import HttpProtocolHandlerPlugin from ..exception import HttpProtocolException, ProxyConnectionFailed from ..parser import HttpParser, httpParserStates, httpParserTypes -# from ..parser.tls import TlsParser, TlsHandshake, tlsContentType, tlsHandshakeType from ...common.types import Readables, Writables from ...common.constants import DEFAULT_CA_CERT_DIR, DEFAULT_CA_CERT_FILE, DEFAULT_CA_FILE From fe8fec68fa1acc68b66f44958d23a0b68548ea5b Mon Sep 17 00:00:00 2001 From: Jerry Date: Wed, 22 Dec 2021 12:36:13 +0800 Subject: [PATCH 14/14] remove unused imports remove unused imports --- proxy/core/connection/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/core/connection/server.py b/proxy/core/connection/server.py index fb201ada1d..7aae5371cc 100644 --- a/proxy/core/connection/server.py +++ b/proxy/core/connection/server.py @@ -11,7 +11,7 @@ import ssl import socket -from typing import Optional, Union, Tuple, Dict +from typing import Optional, Union, Tuple from ...common.utils import new_socket_connection