diff --git a/native/python_package/python_driver/cli.py b/native/python_package/python_driver/cli.py index e75310ed..520d4238 100755 --- a/native/python_package/python_driver/cli.py +++ b/native/python_package/python_driver/cli.py @@ -31,7 +31,7 @@ def get_processor_instance(format_: str, custom_inbuffer: InBuffer=None, """ conf = ProcessorConfigs.get(format_) if not conf: - raise RequestInstantiationException('No RequestProcessor found for format %s' % format) + raise RequestInstantiationException('No RequestProcessor found for format %s' % format_) inbuffer = custom_inbuffer if custom_inbuffer else conf['inbuffer'] outbuffer = custom_outbuffer if custom_outbuffer else conf['outbuffer'] @@ -41,16 +41,7 @@ def get_processor_instance(format_: str, custom_inbuffer: InBuffer=None, def main() -> None: - """ - If you pass the --json command line parameter the replies will be - printed using pprint without wrapping them in the msgpack format which - can be handy when debugging. - """ - - if len(sys.argv) > 1 and sys.argv[1] == '--msgpack': - format_ = 'msgpack' - else: - format_ = 'json' + format_ = 'json' processor, inbuffer = get_processor_instance(format_) try: @@ -58,7 +49,7 @@ def main() -> None: except UnicodeDecodeError: print_exc() print('Error while trying to decode the message, are you sure you are not ' - 'using a different input format that the currently configured (%s)?' % format) + 'using a different input format that the currently configured (%s)?' % format_) if __name__ == '__main__': diff --git a/native/python_package/python_driver/processor_configs.py b/native/python_package/python_driver/processor_configs.py index c419256d..999d0a76 100644 --- a/native/python_package/python_driver/processor_configs.py +++ b/native/python_package/python_driver/processor_configs.py @@ -1,5 +1,5 @@ import sys -from python_driver.requestprocessor import RequestProcessorJSON, RequestProcessorMSGPack +from python_driver.requestprocessor import RequestProcessorJSON ProcessorConfigs = { 'json': { @@ -7,10 +7,4 @@ 'inbuffer': sys.stdin, 'outbuffer': sys.stdout }, - - 'msgpack': { - 'class': RequestProcessorMSGPack, - 'inbuffer': sys.stdin.buffer, - 'outbuffer': sys.stdout.buffer - } } diff --git a/native/python_package/python_driver/requestprocessor.py b/native/python_package/python_driver/requestprocessor.py index 24588da2..a4ec0fda 100644 --- a/native/python_package/python_driver/requestprocessor.py +++ b/native/python_package/python_driver/requestprocessor.py @@ -251,54 +251,3 @@ def process_requests(self, inbuffer: InStrBuffer) -> None: """ for doc in self._extract_docs(inbuffer): self.process_request(doc) - - -class RequestProcessorMSGPack(RequestProcessor): - """ - RequestProcessor subclass that operates deserializing and serializing responses - using the MSGPACK format. Input and output packages - """ - def __init__(self, outbuffer: OutBytesBuffer) -> None: - """ - :param outbuffer: the output buffer. This must be a bytes-based file like object - supporting the write(bytes) and flush() methods. - """ - super().__init__(outbuffer) - - def _send_response(self, response: Response) -> None: - from msgpack import dumps - self.outbuffer.write(dumps(response)) - self.outbuffer.flush() - - def _tostr_request(self, request: RawRequest) -> Request: - """ - Convert all byte-string keys and values to normal strings (non recursively since - we only have one level) - - :param request: dictionary that potentially can contain bytestring keys and values. - :return: the converted dictionary - """ - try: - newrequest = Request({}) - for key, value in request.items(): - if isinstance(value, bytes): - value = value.decode() - - if isinstance(key, bytes): - key = key.decode() - - newrequest[key] = value - except AttributeError as exc: - self.errors.append('Error trying to decode message, are you sure that the input ' + - 'format is msgpack?') - raise AttributeError from exc - return newrequest - - def process_requests(self, inbuffer: InStrBuffer) -> None: - """ - :param inbuffer: file-like object based on bytes supporting the read() and - iteration by lines - """ - from msgpack import Unpacker - for request in Unpacker(inbuffer): - self.process_request(request) diff --git a/native/python_package/requirements.txt b/native/python_package/requirements.txt index 44e0fa00..b464f2e3 100644 --- a/native/python_package/requirements.txt +++ b/native/python_package/requirements.txt @@ -1,4 +1,3 @@ -msgpack-python==0.4.8 pydetector==0.8.1 -e git+git://github.com/python/mypy.git@0bb2d1680e8b9522108b38d203cb73021a617e64#egg=mypy-lang typed-ast==1.0.1 diff --git a/native/python_package/setup.py b/native/python_package/setup.py index 4f974c2e..225cac69 100644 --- a/native/python_package/setup.py +++ b/native/python_package/setup.py @@ -31,7 +31,6 @@ ] }, install_requires=[ - "msgpack-python==0.4.8", "pydetector==0.8.1" ], classifiers=[ diff --git a/native/python_package/test/test_python_driver.py b/native/python_package/test/test_python_driver.py index abf86895..c89f1f85 100644 --- a/native/python_package/test/test_python_driver.py +++ b/native/python_package/test/test_python_driver.py @@ -1,56 +1,20 @@ import io +import json import os +import subprocess import sys -import json import unittest -import subprocess from os.path import join, abspath, dirname -# msgpack is an optional dependency -try: - import msgpack -except ImportError: - TEST_MSGPACK = False -else: - TEST_MSGPACK = True - sys.path.append('..') from python_driver import __version__, get_processor_instance from python_driver.requestprocessor import ( Request, Response, RequestProcessorJSON, InBuffer, EmptyCodeException) -if TEST_MSGPACK: - from python_driver.requestprocessor import RequestProcessorMSGPack from typing import Dict, Any, List, AnyStr, Optional, Iterator, cast CURDIR = abspath(dirname(__file__)) - -def convert_bytes(data: Any, to_bytes: bool=False) -> Any: - """ - Both in normal operation and with this tests data comes trough a bytestream so this is needed to - recursively convert the msgpack incoming data (the pprint data is converted just decoding and using - literal_eval) - """ - if type(data) in (list, tuple): - newlist: List[Any] = [] - for item in data: - newlist.append(convert_bytes(item, to_bytes)) - return newlist - elif isinstance(data, dict): - newdict: Dict[str, Any] = {} - for key, value in data.items(): - newvalue = convert_bytes(value, to_bytes) - newkey = convert_bytes(key, to_bytes) - newdict[newkey] = newvalue - return newdict - elif isinstance(data, bytes) and not to_bytes: - return data.decode() - elif isinstance(data, str) and to_bytes: - return data.encode() - return data - - class TestTypeCheck(unittest.TestCase): def test_10_check(self) -> None: prevdir = os.getcwd() @@ -64,7 +28,7 @@ def test_10_check(self) -> None: class TestPythonDriverBase(unittest.TestCase): def _restart_data(self, format_: str='json') -> None: - assert format_ in ('json', 'msgpack') + assert format_ == 'json' with open(join(CURDIR, 'data', 'helloworld.py')) as f: testcode = f.read() @@ -94,27 +58,21 @@ def _extract_docs(inbuffer: InBuffer) -> Iterator[Response]: yield json.loads(line) def _loadResults(self, format_: str) -> List[Response]: - """Read all msgpacks from the recvbuffer""" + """Read all msgs from the recvbuffer""" self.recvbuffer.seek(0) res: List[Response] = [] - if format_ == 'json': - res = [doc for doc in self._extract_docs(self.recvbuffer)] - elif TEST_MSGPACK and format_ == 'msgpack': - res = [convert_bytes(msg) for msg in msgpack.Unpacker(self.recvbuffer)] + res = [doc for doc in self._extract_docs(self.recvbuffer)] return res class Test10ProcessRequestFunc(TestPythonDriverBase): def _add_to_buffer(self, count: int, format_: str) -> None: - """Add count test msgpacks to the sendbuffer""" + """Add count test msgs to the sendbuffer""" for i in range(count): msg = '' - if TEST_MSGPACK and format_ == 'msgpack': - msg = msgpack.dumps(self.data) - elif format_ == 'json': - msg = json.dumps(self.data, ensure_ascii=False) + '\n' + msg = json.dumps(self.data, ensure_ascii=False) + '\n' self.sendbuffer.write(msg) self.sendbuffer.flush() @@ -176,26 +134,13 @@ def test_010_normal_json(self) -> None: self.assertEqual(len(replies), 1) self._check_reply_dict(replies[0]) - if TEST_MSGPACK: - def test_020_normal_msgpack(self) -> None: - replies = self._send_receive(1, 'msgpack') - self.assertEqual(len(replies), 1) - self._check_reply_dict(replies[0]) - - def test_030_normal_json_many(self) -> None: + def test_020_normal_json_many(self) -> None: replies = self._send_receive(100, 'json') self.assertEqual(len(replies), 100) for reply in replies: self._check_reply_dict(reply) - if TEST_MSGPACK: - def test_040_normal_msgpack_many(self) -> None: - replies = self._send_receive(100, 'msgpack') - self.assertEqual(len(replies), 100) - for reply in replies: - self._check_reply_dict(reply) - - def test_050_error_print(self) -> None: + def test_030_error_print(self) -> None: wrongcode = 'wtf lol' replies = self._send_receive(1, 'json', {'content': wrongcode}) @@ -209,31 +154,7 @@ def test_050_error_print(self) -> None: replies = self._send_receive(1, 'json') self.assertEqual(len(replies), 1) - if TEST_MSGPACK: - def test_060_error_msgpack(self) -> None: - wrongcode = 'wtf lol' - - replies = self._send_receive(1, 'msgpack', {'content': wrongcode}) - self.assertEqual(len(replies), 1) - ast = replies[0].get('ast') - self.assertIsNone(ast) - self._check_reply_dict(replies[0], has_errors=True) - - # Check that it still alive - self._restart_data() - replies = self._send_receive(1, 'json') - self.assertEqual(len(replies), 1) - - def test_070_broken_msgpack(self) -> None: - self._restart_data('msgpack') - brokendata = msgpack.dumps(self.data)[:-30] - self.sendbuffer.write(brokendata) - self.sendbuffer.flush() - reply = self._send_receive(1, 'msgpack', restart_data=False)[0] - self.assertEqual(reply['status'], 'fatal') - self.assertEqual(len(reply['errors']), 1) - - def test_080_broken_json(self) -> None: + def test_040_broken_json(self) -> None: self._restart_data('json') brokendata = json.dumps(self.data, ensure_ascii=False)[:-30] self.sendbuffer.write(brokendata) @@ -244,31 +165,8 @@ def test_080_broken_json(self) -> None: class Test20ReqProcMethods(TestPythonDriverBase): - if TEST_MSGPACK: - def test_10_check_input(self) -> None: - self._restart_data('json') - brequest = convert_bytes(self.data, to_bytes=True) - processor = RequestProcessorMSGPack(self.recvbuffer) - res = processor._parse_input_request(brequest) - self.assertEqual(res[1], 'test.py') - - def test_20_check_input_bad(self) -> None: - self._restart_data('msgpack') - del self.data['content'] - brequest = convert_bytes(self.data, to_bytes=True) - processor = RequestProcessorMSGPack(self.recvbuffer) - with self.assertRaises(EmptyCodeException) as _: # noqa: F841 - processor._parse_input_request(brequest) - - def test_30_send_response_msgpack(self) -> None: - self._restart_data('msgpack') - processor = RequestProcessorMSGPack(self.recvbuffer) - processor._send_response(cast(Response, self.data)) - res = self._loadResults('msgpack') - self.assertEqual(len(res), 1) - self.assertDictEqual(self.data, res[0]) - - def test_40_send_response_json(self) -> None: + + def test_10_send_response_json(self) -> None: self._restart_data('json') processor = RequestProcessorJSON(self.recvbuffer) processor._send_response(cast(Response, self.data)) @@ -278,7 +176,7 @@ def test_40_send_response_json(self) -> None: # process request already tested with TestPythonDriverBase - def test_50_return_error(self) -> None: + def test_20_return_error(self) -> None: self._restart_data('json') processor = RequestProcessorJSON(self.recvbuffer) processor.errors = ['test error']