From 148f41acef7c62cbc948714faac79a0c237224e3 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Fri, 7 Jun 2019 16:52:26 -0700 Subject: [PATCH 1/6] Add sbp2json to libsbp --- .travis.yml | 1 - python/bin/sbp2json | 6 ++ python/requirements.txt | 1 + python/sbp/sbp2json.py | 195 ++++++++++++++++++++++++++++++++++++ python/sbp2json/__main__.py | 4 + python/setup.py | 4 +- test_data/format-test.sh | 4 +- 7 files changed, 211 insertions(+), 4 deletions(-) create mode 100755 python/bin/sbp2json create mode 100644 python/sbp/sbp2json.py create mode 100644 python/sbp2json/__main__.py diff --git a/.travis.yml b/.travis.yml index a80fa5c532..e0a47f22c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,6 @@ matrix: - sudo apt-get install python3.5 python3.5-dev - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - sudo pip install tox - - git clone -b master https://github.com/swift-nav/piksi_tools.git ../piksi_tools script: | pushd haskell docker build -t sbp2json . diff --git a/python/bin/sbp2json b/python/bin/sbp2json new file mode 100755 index 0000000000..55d4ae864a --- /dev/null +++ b/python/bin/sbp2json @@ -0,0 +1,6 @@ +#!/usr/bin/env python + +from sbp.sbp2json import module_main + +if __name__ == "__main__": + module_main() diff --git a/python/requirements.txt b/python/requirements.txt index 19c7a11734..d86d8b1dd4 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -5,3 +5,4 @@ pyserial requests>=2.8.1 numpy==1.16.2 pybase64 +ujson diff --git a/python/sbp/sbp2json.py b/python/sbp/sbp2json.py new file mode 100644 index 0000000000..3fedf8b51e --- /dev/null +++ b/python/sbp/sbp2json.py @@ -0,0 +1,195 @@ +import os +import sys + +import io + +import numpy as np +import json +import ujson + +import decimal as dec + +from sbp.jit import msg +from sbp.jit.table import dispatch + +from sbp import msg as msg_nojit +from sbp.table import dispatch as dispatch_nojit + +NORM = os.environ.get('NOJIT') is not None + +dec.getcontext().rounding = dec.ROUND_HALF_UP + + +def base_cl_options(): + import argparse + parser = argparse.ArgumentParser(prog="sbp2json", description="Swift Navigation SBP to JSON parser") + parser.add_argument('--mode', type=str, choices=['json', 'ujson'], default='ujson') + + group_json = parser.add_argument_group('json specific arguments') + group_json.add_argument( + "--judicious-rounding", + action="store_true", + help="Use Numpy's judicious rounding and reprentation precision. Only on Python 3.5 and forward.") + group_json.add_argument( + "--sort-keys", + action="store_true", + help="Sort JSON log elements by keys") + + return parser + + +def get_args(): + """ + Get and parse arguments. + """ + parser = base_cl_options() + args = parser.parse_args() + + if args.mode == 'ujson' and len(sys.argv) > 3: + print('ERROR: ujson mode does not support given arguments') + parser.print_help() + return None + + if args.judicious_rounding and sys.version_info[0] < 3: + print('ERROR: Must be using Python 3.5 or newer for --float-meta') + parser.print_help() + return None + + return args + + +class SbpJSONEncoder(json.JSONEncoder): + # Overwrite for json.JSONEncoder.iterencode() + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + For example:: + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = json.encoder.encode_basestring_ascii + else: + _encoder = json.encoder.encode_basestring + + def floatstr(o, allow_nan=self.allow_nan, + _repr=float.__repr__, _inf=float('inf'), _neginf=-float('inf')): + # Check for specials. Note that this type of test is processor + # and/or platform-specific, so do tests which don't depend on the + # internals. + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + elif o.is_integer(): + return str(int(o)) + elif abs(o) < 0.1 or abs(o) > 9999999: + # GHC uses showFloat to print which will result in the + # scientific notation whenever the absolute value is outside the + # range between 0.1 and 9,999,999. Numpy wants to put '+' after + # exponent sign, strip it. Use decimal module to control + # rounding method. + text = np.format_float_scientific(o, precision=None, unique=True, trim='0', exp_digits=1) + d = dec.Decimal(text) + rounded = round(dec.Decimal(o), abs(d.as_tuple().exponent)) + + if d == rounded: + # original is good + return text.replace('+', '') + + return ('{:.' + str(len(d.as_tuple().digits) - 1) + 'e}').format(rounded).replace('+', '') + else: + d = dec.Decimal(np.format_float_positional(o, precision=None, unique=True, trim='0')) + return round(dec.Decimal(o), abs(d.as_tuple().exponent)).to_eng_string() + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + _iterencode = json.encoder._make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot) + return _iterencode(o, 0) + + +def dump(args, res): + if 'json' == args.mode: + sys.stdout.write(json.dumps(res, + allow_nan=False, + sort_keys=args.sort_keys, + separators=(',', ':'), + cls=SbpJSONEncoder if args.judicious_rounding else None)) + elif 'ujson' == args.mode: + sys.stdout.write(ujson.dumps(res)) + + sys.stdout.write("\n") + + +def sbp_main(args): + msg.SBP.judicious_rounding = args.judicious_rounding + + header_len = 6 + reader = io.open(sys.stdin.fileno(), 'rb') + buf = np.zeros(4096, dtype=np.uint8) + unconsumed_offset = 0 + read_offset = 0 + buffer_remaining = len(buf) + while True: + if buffer_remaining == 0: + memoryview(buf)[0:(read_offset - unconsumed_offset)] = \ + memoryview(buf)[unconsumed_offset:read_offset] + read_offset = read_offset - unconsumed_offset + unconsumed_offset = 0 + buffer_remaining = len(buf) - read_offset + mv = memoryview(buf)[read_offset:] + read_length = reader.readinto(mv) + if read_length == 0: + unconsumed = read_offset - unconsumed_offset + if unconsumed != 0: + sys.stderr.write("unconsumed: {}\n".format(unconsumed)) + sys.stderr.flush() + break + read_offset += read_length + buffer_remaining -= read_length + while True: + if NORM: + from construct.core import StreamError + bytes_available = read_offset - unconsumed_offset + b = buf[unconsumed_offset:(unconsumed_offset + bytes_available)] + try: + m = msg_nojit.SBP.unpack(b) + except StreamError: + break + m = dispatch_nojit(m) + dump(args, m) + consumed = header_len + m.length + 2 + else: + consumed, payload_len, msg_type, sender, crc, crc_fail = \ + msg.unpack_payload(buf, unconsumed_offset, (read_offset - unconsumed_offset)) + + if not crc_fail and msg_type != 0: + payload = buf[unconsumed_offset + header_len:unconsumed_offset + header_len + payload_len] + m = dispatch(msg_type)(msg_type, sender, payload_len, payload, crc) + res, offset, length = m.unpack(buf, unconsumed_offset + header_len, payload_len) + dump(args, res) + + if consumed == 0: + break + unconsumed_offset += consumed + + +def module_main(): + args = get_args() + if not args: + sys.exit(1) + sbp_main(args) diff --git a/python/sbp2json/__main__.py b/python/sbp2json/__main__.py new file mode 100644 index 0000000000..c4e5802b16 --- /dev/null +++ b/python/sbp2json/__main__.py @@ -0,0 +1,4 @@ +from sbp.sbp2json import module_main + +if __name__ == "__main__": + module_main() diff --git a/python/setup.py b/python/setup.py index 9f10366612..11dce42952 100755 --- a/python/setup.py +++ b/python/setup.py @@ -36,6 +36,7 @@ 'sbp.client.drivers', 'sbp.client.loggers', 'sbp.client.util', + 'sbp2json', ] PLATFORMS = [ @@ -198,4 +199,5 @@ def write_version_py(filename=VERSION_PY_PATH): tests_require=TEST_REQUIRES, use_2to3=False, zip_safe=False, - ext_modules=ext_modules) + ext_modules=ext_modules, + scripts=['bin/sbp2json']) diff --git a/test_data/format-test.sh b/test_data/format-test.sh index f514660be6..b3cf373c59 100755 --- a/test_data/format-test.sh +++ b/test_data/format-test.sh @@ -16,7 +16,7 @@ OUTPUT_LONG_HS=$TESTDATA_ROOT/long_hask_pretty.json OUTPUT_LONG_PY=$TESTDATA_ROOT/long_py_pretty.json PYTHONPATH=$TESTDATA_ROOT/../python/ \ - python $TESTDATA_ROOT/../../piksi_tools/piksi_tools/sbp2json.py \ + python $TESTDATA_ROOT/../bin/sbp2json.py \ < $INPUT_SHORT --mode json --sort-keys --judicious-rounding > $OUTPUT_SHORT_PY if [ ! -f $OUTPUT_SHORT_HS ]; then @@ -28,7 +28,7 @@ diff $OUTPUT_SHORT_HS $OUTPUT_SHORT_PY || exit 1 echo -e "Sanity check \e[32mOK\e[0m, please wait for format test.." PYTHONPATH=$TESTDATA_ROOT/../python/ \ - python $TESTDATA_ROOT/../../piksi_tools/piksi_tools/sbp2json.py \ + python $TESTDATA_ROOT/../bin/sbp2json.py \ < $INPUT_LONG --mode json --sort-keys --judicious-rounding > $OUTPUT_LONG_PY if [ ! -f $OUTPUT_LONG_HS ]; then From db2a527a420e7004a4ea2ff222baca7e71556db3 Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Fri, 7 Jun 2019 16:58:39 -0700 Subject: [PATCH 2/6] Update format-test.sh --- test_data/format-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_data/format-test.sh b/test_data/format-test.sh index b3cf373c59..1ff234e197 100755 --- a/test_data/format-test.sh +++ b/test_data/format-test.sh @@ -16,7 +16,7 @@ OUTPUT_LONG_HS=$TESTDATA_ROOT/long_hask_pretty.json OUTPUT_LONG_PY=$TESTDATA_ROOT/long_py_pretty.json PYTHONPATH=$TESTDATA_ROOT/../python/ \ - python $TESTDATA_ROOT/../bin/sbp2json.py \ + python $TESTDATA_ROOT/../python/bin/sbp2json.py \ < $INPUT_SHORT --mode json --sort-keys --judicious-rounding > $OUTPUT_SHORT_PY if [ ! -f $OUTPUT_SHORT_HS ]; then @@ -28,7 +28,7 @@ diff $OUTPUT_SHORT_HS $OUTPUT_SHORT_PY || exit 1 echo -e "Sanity check \e[32mOK\e[0m, please wait for format test.." PYTHONPATH=$TESTDATA_ROOT/../python/ \ - python $TESTDATA_ROOT/../bin/sbp2json.py \ + python $TESTDATA_ROOT/../python/bin/sbp2json.py \ < $INPUT_LONG --mode json --sort-keys --judicious-rounding > $OUTPUT_LONG_PY if [ ! -f $OUTPUT_LONG_HS ]; then From 33a60c7d4a0324461da52a678f565507c092154f Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Fri, 7 Jun 2019 17:28:56 -0700 Subject: [PATCH 3/6] Update format-test.sh --- test_data/format-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_data/format-test.sh b/test_data/format-test.sh index 1ff234e197..271e7915d7 100755 --- a/test_data/format-test.sh +++ b/test_data/format-test.sh @@ -16,7 +16,7 @@ OUTPUT_LONG_HS=$TESTDATA_ROOT/long_hask_pretty.json OUTPUT_LONG_PY=$TESTDATA_ROOT/long_py_pretty.json PYTHONPATH=$TESTDATA_ROOT/../python/ \ - python $TESTDATA_ROOT/../python/bin/sbp2json.py \ + python $TESTDATA_ROOT/../python/bin/sbp2json \ < $INPUT_SHORT --mode json --sort-keys --judicious-rounding > $OUTPUT_SHORT_PY if [ ! -f $OUTPUT_SHORT_HS ]; then @@ -28,7 +28,7 @@ diff $OUTPUT_SHORT_HS $OUTPUT_SHORT_PY || exit 1 echo -e "Sanity check \e[32mOK\e[0m, please wait for format test.." PYTHONPATH=$TESTDATA_ROOT/../python/ \ - python $TESTDATA_ROOT/../python/bin/sbp2json.py \ + python $TESTDATA_ROOT/../python/bin/sbp2json \ < $INPUT_LONG --mode json --sort-keys --judicious-rounding > $OUTPUT_LONG_PY if [ ! -f $OUTPUT_LONG_HS ]; then From 8d8a9f050e445bde9c4a9f4ffa112100adf1abde Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Fri, 7 Jun 2019 18:11:23 -0700 Subject: [PATCH 4/6] Enable debugging for benchmark.sh --- test_data/benchmark.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test_data/benchmark.sh b/test_data/benchmark.sh index d9b9aa863f..d25aa994ea 100755 --- a/test_data/benchmark.sh +++ b/test_data/benchmark.sh @@ -1,10 +1,11 @@ #!/usr/bin/env bash + if [ "$#" -ne 1 ]; then echo "Skipping benchmark.sh, enable by providing a full path to Haskell SBP tools" exit 0 fi -set -e +set -ex TESTDATA_ROOT=$(git rev-parse --show-toplevel)/test_data echo "Running benchmark, please wait.." From eac6f0c88b358874616a6959c7cd0d3a59091c0d Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sat, 8 Jun 2019 08:32:18 -0700 Subject: [PATCH 5/6] Update benchmark.sh --- test_data/benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_data/benchmark.sh b/test_data/benchmark.sh index d25aa994ea..fab64511ce 100755 --- a/test_data/benchmark.sh +++ b/test_data/benchmark.sh @@ -12,7 +12,7 @@ echo "Running benchmark, please wait.." # http://mywiki.wooledge.org/BashFAQ/032 time_py=$(TIMEFORMAT="%R"; { time PYTHONPATH=$TESTDATA_ROOT/../python/ \ - python $TESTDATA_ROOT/../../piksi_tools/piksi_tools/sbp2json.py \ + python $TESTDATA_ROOT/../python/bin/sbp2json \ < $TESTDATA_ROOT/long.sbp --mode ujson > $TESTDATA_ROOT/long_py.json; } 2>&1) echo "Python" $time_py From c0ad579bcb841caab5c3ea616565f9ab11e4a9dd Mon Sep 17 00:00:00 2001 From: Jason Mobarak Date: Sat, 8 Jun 2019 09:19:15 -0700 Subject: [PATCH 6/6] Update benchmark.sh --- test_data/benchmark.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_data/benchmark.sh b/test_data/benchmark.sh index fab64511ce..02094f261e 100755 --- a/test_data/benchmark.sh +++ b/test_data/benchmark.sh @@ -5,7 +5,7 @@ if [ "$#" -ne 1 ]; then exit 0 fi -set -ex +set -e TESTDATA_ROOT=$(git rev-parse --show-toplevel)/test_data echo "Running benchmark, please wait.."