diff --git a/.travis.yml b/.travis.yml index a80fa5c532..201e9ee790 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ matrix: - sudo apt-get install python3.5 python3.5-dev - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce - sudo pip install tox - - git clone -b master https://github.com/swift-nav/piksi_tools.git ../piksi_tools + - git clone -b pmiettinen/esd-1156-numba-deployment https://github.com/swift-nav/piksi_tools.git ../piksi_tools script: | pushd haskell docker build -t sbp2json . diff --git a/HOWTO.md b/HOWTO.md index 4ba1e2cd93..ff315d78a3 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -144,17 +144,54 @@ Ubuntu 16.04. [GitHub](https://github.com/swift-nav/libsbp/releases) and add the RELEASE_NOTES.md. -7. Distribute release packages: `make dist`. You may need credentials - on the appropriate package repositories. Ignore the GPG error in `stack`, - the package will get uploaded correctly anyway. If the release is - a Python only change it may be appropriate to just publish to PyPI - with `make dist-python` -- we typically update all other supported - languages when we make an official firmware release. +7. Distribute release packages. You can attempt to run all releases + with `make dist` -- this will likely not work through... it is + advisable to run each dist target separately. In particular: + + - `make dist-javascript` + - `make dist-haskell` + - `make dist-pdf` + - `make dist-python` (see section on Python below) + + You may need credentials on the appropriate package repositories. Ignore the + GPG error in `stack`, the package will get uploaded correctly anyway. If + the release is a Python only change it may be appropriate to just publish to + PyPI with `make dist-python` (see section on Python below) -- we typically + update all other supported languages when we make an official firmware + release. 8. Releases are not only never perfect, they never really end. Please pay special attention to any downstream projects or users that may have issues or regressions as a consequence of the release version. +# Distributing Python + +Python distribution requires compilation for the JIT accelerated `sbp.jit` +package. This package uses the Python `numba` library, which supports AOT +compilation of a native Python extension. The distributions for each platform +can be created by running the `make dist-python` target on each platform +(Windows, Mac OS X, Linux x86, and Linux ARM through docker). + +For example, running this: +``` +make dist-python PYPI_USERNAME=swiftnav PYPI_PASSWORD=... +``` + +...will produce and upload a `.whl` appropriate for that platform. A +wheel that targets any platform (but requires that `numba` be installed) +can be produced and uploaded by running the following command: +``` +make dist-python PYPI_USERNAME=swiftnav PYPI_PASSWORD=... LIBSBP_BUILD_ANY=y +``` + +The Linux ARM build of libsbp can be done either natively, or through docker +via the following set of command: +``` +docker build -f python/Dockerfile.arm -t libsbp-arm . +docker run -v $PWD:/work --rm -it libsbp-arm /bin/bash +make dist-python PYPI_USERNAME=swiftnav PYPI_PASSWORD=... +``` + # Contributions This library is developed internally by Swift Navigation. We welcome diff --git a/Makefile b/Makefile index f6ee4894b7..c96d4fa979 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # before using it to do Crazy Things. SHELL := /bin/bash -SWIFTNAV_ROOT := $(shell pwd) +SWIFTNAV_ROOT := $(CURDIR) MAKEFLAGS += SWIFTNAV_ROOT=$(SWIFTNAV_ROOT) SBP_SPEC_DIR := $(SWIFTNAV_ROOT)/spec/yaml/swiftnav/sbp/ SBP_TESTS_SPEC_DIR := $(SWIFTNAV_ROOT)/spec/tests/yaml/ @@ -249,15 +249,22 @@ dist-python: make -C $(SWIFTNAV_ROOT)/python SBP_VERSION="$(SBP_MAJOR_VERSION).$(SBP_MINOR_VERSION).$(SBP_PATCH_VERSION)" deploy $(call announce-end,"Finished deploying Python package") -dist: dist-python - $(call announce-begin,"Deploying packages") +dist-javascript: + $(call announce-begin,"Deploying Javascript package") npm publish - pushd $(SWIFTNAV_ROOT)/haskell - stack sdist - stack upload . - popd + $(call announce-begin,"Finished deploying Javascript package") + +dist-haskell: + $(call announce-begin,"Deploying Haskell package") + (cd $(SWIFTNAV_ROOT)/haskell; stack sdist; stack upload .) + $(call announce-begin,"Finished deploying Haskell package") + +dist-pdf: + $(call announce-begin,"Deploying PDF documentation") make pdf_dist - $(call announce-end,"Finished deploying packages") + $(call announce-begin,"Finished deploying PDF documentation") + +dist: dist-python dist-javascript dist-haskell dist-pdf pdf: $(call announce-begin,"Generating PDF datasheet documentation") diff --git a/python/Dockerfile.arm b/python/Dockerfile.arm new file mode 100644 index 0000000000..1875701867 --- /dev/null +++ b/python/Dockerfile.arm @@ -0,0 +1,14 @@ +FROM balenalib/armv7hf-debian:sid-build + +RUN [ "cross-build-start" ] + +RUN \ + echo Setting up ARM build environment... \ + && apt-get update \ + && apt-get install wget bzip2 build-essential llvm-6.0-dev python3 \ + && update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-6.0 1 \ + && wget -O /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-armv7l.sh \ + && bash /tmp/miniconda.sh -b \ + && rm /tmp/miniconda.sh + +ENV PATH=/root/miniconda3/bin:$PATH diff --git a/python/MANIFEST.in b/python/MANIFEST.in index c626bc8882..c55c64ca54 100644 --- a/python/MANIFEST.in +++ b/python/MANIFEST.in @@ -9,5 +9,5 @@ include .gitignore include LICENSE include tox.ini include sbp/RELEASE-VERSION -recursive-include sbp/ *.py +recursive-include sbp *.py prune docs/_build diff --git a/python/Makefile b/python/Makefile index 0761c23eb7..5bb4feeb47 100644 --- a/python/Makefile +++ b/python/Makefile @@ -1,8 +1,14 @@ - .PHONY: deploy -DEPLOY_PYTHON := $(CURDIR)/deploy.bash -DEPLOY_COMMAND := SBP_VERSION=$(SBP_VERSION) $(SHELL) $(DEPLOY_PYTHON) +ifeq ($(OS),Windows_NT) +PYTHON := python +else +PYTHON := python3 +endif + +DEPLOY_PYTHON := $(CURDIR)/deploy.py +DEPLOY_COMMAND := $(PYTHON) $(DEPLOY_PYTHON) +deploy: export SBP_VERSION=$(SBP_VERSION) deploy: $(DEPLOY_COMMAND) diff --git a/python/deploy.bash b/python/deploy.bash deleted file mode 100755 index e51e666baa..0000000000 --- a/python/deploy.bash +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -IFS=$'\n\t' - -[[ -n "$PYPI_USERNAME" ]] || \ - { printf "\n!!! Please set PYPI_USERNAME in the environment !!!\n\n"; exit 1; } - -[[ -n "$PYPI_PASSWORD" ]] || \ - { printf "\n!!! Please set PYPI_PASSWORD in the environment !!!\n\n"; exit 1; } - -if ! command -v conda; then - echo '!!! Please install conda to deploy python !!!' -fi - -conda_dir=$(mktemp -d) -conda create --yes -p "$conda_dir" python=3.5 - -# Activate conda -{ - # Workaround bug in activate code... - export PS1='' - - eval "$(conda shell.bash hook)" - # shellcheck disable=SC1091 - source activate "$conda_dir" -} - -conda install --yes \ - cython virtualenv twine wheel - -deploy_dir=$(mktemp -d) -trap 'rm -rf "$deploy_dir" "$conda_dir"' EXIT - -echo "$deploy_dir" -cd "$deploy_dir" - -echo ">>> Building staging area for deployment ..." - -mkdir module - -cp -r "$(dirname "$0")"/../.git . - -cp -r "$(dirname "$0")"/.coveragerc module/. -cp -r "$(dirname "$0")"/.gitignore module/. - -cp -r "$(dirname "$0")"/* module/. - -echo ">>> Pruning ..." -rm -r -f module/docs/_build -rm -r -f module/build/* - -echo ">>> Patching setup.py ..." -sed -i.backup 's@IS_RELEASED = False@IS_RELEASED = True@' module/setup.py - -cd module - -echo ">>> Building Python wheel ..." -python setup.py sdist bdist_wheel - -echo ">>> Uploading Python wheel ..." -twine upload -u "$PYPI_USERNAME" -p "$PYPI_PASSWORD" "dist/sbp-$SBP_VERSION-*.whl" diff --git a/python/deploy.py b/python/deploy.py new file mode 100644 index 0000000000..256660794b --- /dev/null +++ b/python/deploy.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python + +import os +import sys +import glob +import shutil +import platform +import tempfile +import subprocess + +if 'PYPI_USERNAME' not in os.environ: + print("\n!!! Please set PYPI_USERNAME in the environment !!!\n\n") + sys.exit(1) + +PYPI_USERNAME = os.environ['PYPI_USERNAME'] + +if 'PYPI_PASSWORD' not in os.environ: + print("\n!!! Please set PYPI_PASSWORD in the environment !!!\n\n") + sys.exit(1) + +PYPI_PASSWORD = os.environ['PYPI_PASSWORD'] + +if 'SBP_VERSION' not in os.environ: + print("\n!!! Please set SBP_VERSION in the environment !!!\n\n") + sys.exit(1) + +SBP_VERSION = os.environ['SBP_VERSION'] + +USE_TEST_PYPI = bool(os.environ.get('USE_TEST_PYPI', None)) + +if not shutil.which('conda'): + print("\n!!! Please install conda to deploy python !!!\n\n") + sys.exit(1) + +script_dir = os.path.dirname(os.path.abspath(__file__)) +repo_dir = os.path.join(script_dir, "..") + +os.chdir(script_dir) + +if platform.system() == "Linux": + DASHDASH = ["--"] +else: + DASHDASH = [] + +def twine_upload(conda_dir, wheel, use_conda=True): + + cmd_prefix = ["/usr/bin/python3", "-m"] + if use_conda: + cmd_prefix = ["conda", "run", "-p", conda_dir] + DASHDASH + + invoke = subprocess.check_call if not USE_TEST_PYPI else subprocess.call + ret = invoke(cmd_prefix + [ + "twine", "upload", "-u", PYPI_USERNAME, "-p", PYPI_PASSWORD] + ([ + "--repository-url", "https://test.pypi.org/legacy/"] + if USE_TEST_PYPI else [] + ) + [wheel]) + if USE_TEST_PYPI and ret != 0: + print(">>> Warning: twine upload returned exit code {}".format(ret)) + + +def build_wheel_native(conda_dir, deploy_dir, py_version): + + print(">>> Installing native deps for: {}...".format(py_version)) + + subprocess.check_call(["apt-get", "update"]) + + subprocess.check_call(["apt-get", "install", "-y", + "python3", "python3-wheel", "cython3", "python3-pip", "python3-dev", + ]) + + subprocess.check_call([ + "/usr/bin/python3", "-m", + "pip", "install", "--upgrade", "pip" + ]) + + subprocess.check_call([ + "/usr/bin/python3", "-m", + "pip", "install", "twine", "numpy", "setuptools" + ]) + + print(">>> Installing setup deps in Python {} conda environment...".format(py_version)) + + subprocess.check_call([ + "/usr/bin/python3", "-m", + "pip", "install", "--ignore-installed", "-r", "setup_requirements.txt" + ]) + + run_bdist(conda_dir, deploy_dir, py_version, use_conda=False) + + +def invoke_bdist(conda_dir, use_conda): + + cmd_prefix = ["/usr/bin/python3"] + if use_conda: + cmd_prefix = ["conda", "run", "-p", conda_dir] + DASHDASH + ["python"] + + subprocess.check_call(cmd_prefix + [ + "setup.py", "bdist_wheel" + ]) + + +def run_bdist(conda_dir, deploy_dir, py_version, use_conda=True): + + print(">>> Building staging area for deployment ...") + + os.chdir(deploy_dir) + os.mkdir('module') + + shutil.copytree(os.path.join(repo_dir, ".git"), ".git") + + shutil.copy(os.path.join(script_dir, ".coveragerc"), "module/.coveragerc") + shutil.copy(os.path.join(script_dir, ".gitignore"), "module/.gitignore") + shutil.copy(os.path.join(script_dir, ".flake8"), "module/.flake8") + + for dirent in glob.glob(os.path.join(script_dir, "*")): + _, leaf_name = os.path.split(dirent) + if os.path.isdir(dirent): + print('Copying (recursive) {}'.format(dirent)) + shutil.copytree(dirent, os.path.join("module", leaf_name)) + else: + print('Copying (non-recursive) {}'.format(dirent)) + shutil.copy(dirent, os.path.join("module", leaf_name)) + + print(">>> Pruning ...") + + if os.path.exists("module/docs/_build"): + shutil.rmtree("module/docs/_build") + + for dirent in glob.glob("module/build/*"): + shutil.rmtree(dirent) if os.path.isdir(dirent) else os.unlink(dirent) + + with open("module/setup.py", "rb") as fp: + data = fp.read() + with open("module/setup.py", "wb") as fp: + fp.write(data.replace(b"IS_RELEASED = False", b"IS_RELEASED = True")) + + os.chdir("module") + + print(">>> Staged to '{}'...'".format(deploy_dir)) + + print(">>> Building Python wheel ...") + + invoke_bdist(conda_dir, use_conda) + + whl_pattern = "dist/sbp-{}-*.whl".format(SBP_VERSION) + print(">>> Uploading Python wheel (glob: {})...".format(whl_pattern)) + + wheels = glob.glob(whl_pattern) + if not wheels: + print("\n!!! No Python wheel (.whl) file found...\n\n") + sys.exit(1) + + wheel = wheels[0] + + print(">>> Found wheel (of {} matches): {}".format(len(wheels), wheel)) + + twine_upload(conda_dir, wheel, use_conda) + + +def build_wheel_conda(conda_dir, deploy_dir, py_version): + + print(">>> Creating conda environment for Python version: {}...".format(py_version)) + + subprocess.check_call([ + "conda", "create", "--yes", "-p", conda_dir, + "python={}".format(py_version)]) + + if platform.system() == 'Linux' and platform.machine() == 'x86_64': + subprocess.check_call([ + "conda", "install", "--yes", "-p", conda_dir, + "gcc_linux-64", "gxx_linux-64" + ]) + + print(">>> Installing build deps in Python {} conda environment...".format(py_version)) + + subprocess.check_call([ + "conda", "install", "-p", conda_dir, "--yes", + "cython", "wheel", "setuptools" + ]) + subprocess.check_call([ + "conda", "run", "-p", conda_dir] + DASHDASH + [ + "pip", "install", "--upgrade", "pip" + ]) + subprocess.check_call([ + "conda", "run", "-p", conda_dir] + DASHDASH + [ + "pip", "install", "twine", "numpy" + ]) + + print(">>> Installing setup deps in Python {} conda environment...".format(py_version)) + + subprocess.check_call([ + "conda", "run", "-p", conda_dir] + DASHDASH + [ + "pip", "install", "--ignore-installed", "-r", "setup_requirements.txt" + ]) + + run_bdist(conda_dir, deploy_dir, py_version, use_conda=True) + + +def build_wheel(conda_dir, deploy_dir, py_version): + if platform.system() == "Linux" and platform.machine().startswith("arm") and py_version == "3.7": + build_wheel_native(conda_dir, deploy_dir, py_version) + else: + build_wheel_conda(conda_dir, deploy_dir, py_version) + + +def py_versions(): + if os.environ.get('LIBSBP_BUILD_ANY', None): + return ["3.7"] + if platform.system() == "Linux" and platform.machine().startswith("arm"): + return ["2.7", "3.7"] + else: + return ["2.7", "3.5", "3.7"] + + +for py_version in py_versions(): + + print(">>> Building wheel for Python {}...".format(py_version)) + + conda_tmp_dir = tempfile.mkdtemp() + conda_dir = os.path.join(conda_tmp_dir, "conda") + + deploy_dir = tempfile.mkdtemp() + + try: + build_wheel(conda_dir, deploy_dir, py_version) + finally: + os.chdir(script_dir) + if platform.system() == "Linux" and not platform.machine().startswith("arm"): + shutil.rmtree(conda_tmp_dir) + else: + subprocess.check_call(["rm", "-fr", conda_dir]) + if platform.system() == "Windows": + # Workaround a permission denied error that happens for the copied + # .git directory... + subprocess.check_call(["rmdir", "/s", "/q", deploy_dir], shell=True) + else: + shutil.rmtree(deploy_dir) diff --git a/python/requirements.txt b/python/requirements.txt index befd25a028..19c7a11734 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -3,8 +3,5 @@ pyftdi==0.13.4 pylibftdi pyserial requests>=2.8.1 -llvmlite==0.26.0 numpy==1.16.2 -numba==0.41.0 pybase64 -cffi diff --git a/python/sbp/RELEASE-VERSION b/python/sbp/RELEASE-VERSION index 160fe391c8..da6b0a8f16 100644 --- a/python/sbp/RELEASE-VERSION +++ b/python/sbp/RELEASE-VERSION @@ -1 +1 @@ -2.5.5 \ No newline at end of file +2.5.6 diff --git a/python/sbp/constants.py b/python/sbp/constants.py new file mode 100644 index 0000000000..e1aa34f3b7 --- /dev/null +++ b/python/sbp/constants.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +# Copyright (C) 2019 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +SBP_PREAMBLE = 0x55 + +# Default sender ID. Intended for messages sent from the host to the +# device. +SENDER_ID = 0x42 + +crc16_tab = [ + 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, + 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, + 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, + 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, + 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, + 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, + 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, + 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, + 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, + 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, + 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, + 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, + 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, + 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, + 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, + 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, + 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, + 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, + 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, + 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, + 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, + 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, + 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, + 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, + 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, + 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, + 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, + 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, + 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, + 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, + 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, + 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0] diff --git a/python/sbp/jit/msg.py b/python/sbp/jit/msg.py index 5f5a6053e0..8795ce8bbb 100644 --- a/python/sbp/jit/msg.py +++ b/python/sbp/jit/msg.py @@ -10,159 +10,52 @@ # EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED # WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. +import importlib + from pybase64 import standard_b64encode import decimal as dec import numpy as np -import numba as nb - -import numba.cffi_support - -from sbp.msg import crc16jit -from sbp.msg import SENDER_ID as _SENDER_ID -from sbp.msg import SBP_PREAMBLE as _SBP_PREAMBLE - -from sbp.jit import parse_float -from sbp.jit import parse_float_c - - -numba.cffi_support.register_module(parse_float_c) -_get_f32 = parse_float_c.lib.get_f32 -_get_f64 = parse_float_c.lib.get_f64 +from sbp.constants import SENDER_ID as _SENDER_ID +from sbp.constants import SBP_PREAMBLE as _SBP_PREAMBLE + +from pkgutil import iter_modules + +import sys + +parse_jit_name = "parse_jit_py{}".format(str(sys.version_info[0]) + str(sys.version_info[1])) + +if parse_jit_name in (name for loader, name, ispkg in iter_modules()): + # found in sys.path + parse_jit = importlib.import_module(parse_jit_name) +elif parse_jit_name in (name for loader, name, ispkg in iter_modules(['sbp/jit'])): + # found in sbp.jit + parse_jit = importlib.import_module('sbp.jit.' + parse_jit_name) +else: + # not found -> compile + from sbp.jit import parse + parse.compile() + parse_jit = importlib.import_module('sbp.jit.' + parse_jit_name) + +get_u8 = parse_jit.get_u8 +get_u16 = parse_jit.get_u16 +get_u32 = parse_jit.get_u32 +get_u64 = parse_jit.get_u64 +get_s8 = parse_jit.get_s8 +get_s16 = parse_jit.get_s16 +get_s32 = parse_jit.get_s32 +get_s64 = parse_jit.get_s64 +get_f32 = lambda buf, offset, length: (float(np.frombuffer(buf, dtype=np.float32, count=1, offset=offset)), offset + 4, length - 4) +get_f64 = lambda buf, offset, length: (float(np.frombuffer(buf, dtype=np.float64, count=1, offset=offset)), offset + 8, length - 8) +_get_string = parse_jit._get_string +unpack_payload = parse_jit.unpack_payload SENDER_ID = _SENDER_ID SBP_PREAMBLE = _SBP_PREAMBLE -@nb.jit('Tuple((u1,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_u8(buf, offset, length): - if length < 1: - return (0, offset, length) - return buf[offset], offset + 1, length - 1 - - -@nb.jit('Tuple((u2,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_u16(buf, offset, length): - if length < 2: - return (0, offset, length) - msb = nb.u2(buf[offset + 1]) << 8 - lsb = nb.u2(buf[offset + 0]) << 0 - return msb | lsb, offset + 2, length - 2 - - -@nb.jit('Tuple((u4,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_u32(buf, offset, length): - if length < 4: - return (0, offset, length) - a = nb.u4(buf[offset + 3]) << 24 - b = nb.u4(buf[offset + 2]) << 16 - c = nb.u4(buf[offset + 1]) << 8 - d = nb.u4(buf[offset + 0]) << 0 - return a | b | c | d, offset + 4, length - 4 - - -@nb.jit('Tuple((u8,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_u64(buf, offset, length): - if length < 8: - return (0, offset, length) - a = nb.u8(buf[offset + 7]) << 54 - b = nb.u8(buf[offset + 6]) << 48 - c = nb.u8(buf[offset + 5]) << 40 - d = nb.u8(buf[offset + 4]) << 32 - e = nb.u8(buf[offset + 3]) << 24 - f = nb.u8(buf[offset + 2]) << 16 - g = nb.u8(buf[offset + 1]) << 8 - h = nb.u8(buf[offset + 0]) << 0 - return a | b | c | d | e | f | g | h, offset + 8, length - 8 - - -@nb.jit('Tuple((i1,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_s8(buf, offset, length): - if length < 1: - return (0, offset, length) - return buf[offset], offset + 1, length - 1 - - -@nb.jit('Tuple((i2,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_s16(buf, offset, length): - if length < 2: - return (0, offset, length) - msb = nb.i2(buf[offset + 1]) << 8 - lsb = nb.i2(buf[offset + 0]) << 0 - return msb | lsb, offset + 2, length - 2 - - -@nb.jit('Tuple((i4,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_s32(buf, offset, length): - if length < 4: - return (0, offset, length) - a = nb.i4(buf[offset + 3]) << 24 - b = nb.i4(buf[offset + 2]) << 16 - c = nb.i4(buf[offset + 1]) << 8 - d = nb.i4(buf[offset + 0]) << 0 - return a | b | c | d, offset + 4, length - 4 - - -@nb.jit('Tuple((i8,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_s64(buf, offset, length): - if length < 8: - return (0, offset, length) - a = nb.i8(buf[offset + 7]) << 54 - b = nb.i8(buf[offset + 6]) << 48 - c = nb.i8(buf[offset + 5]) << 40 - d = nb.i8(buf[offset + 4]) << 32 - e = nb.i8(buf[offset + 3]) << 24 - f = nb.i8(buf[offset + 2]) << 16 - g = nb.i8(buf[offset + 1]) << 8 - h = nb.i8(buf[offset + 0]) << 0 - return a | b | c | d | e | f | g | h, offset + 8, length - 8 - - -@nb.jit('Tuple((f4,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_f32(buf, offset, length): - if length < 4: - return (0, offset, length) - res = _get_f32(buf[offset + 0], - buf[offset + 1], - buf[offset + 2], - buf[offset + 3]) - return res, offset + 4, length - 4 - - -@nb.jit('Tuple((f8,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) -def get_f64(buf, offset, length): - if length < 8: - return (0, offset, length) - res = _get_f64(buf[offset + 0], - buf[offset + 1], - buf[offset + 2], - buf[offset + 3], - buf[offset + 4], - buf[offset + 5], - buf[offset + 6], - buf[offset + 7]) - return res, offset + 8, length - 8 - - -@nb.jit('Tuple((u1[:],u4,u4))(u1[:],u4,u4,b1)', nopython=True, nogil=True) -def _get_string(buf_in, offset, length, check_null): - buf_out = np.zeros(256, dtype=np.uint8) - i = nb.u4(0) - null_term = False - while i < length: - if check_null and buf_in[offset + i] == 0: - null_term = True - break - buf_out[i] = buf_in[offset + i] - i = nb.u4(i + nb.u4(1)) - if null_term: - return buf_out[:i], offset + i + 1, i + 1 - else: - return buf_out[:i], offset + i, i - - def judicious_round(f): # Let numpy's judicious rounding tell us the amount of digits we # want as it seems to align with Haskell's output @@ -270,58 +163,6 @@ def __eq__(self, other): except AttributeError: return False - @staticmethod - @nb.jit('Tuple((u4, u2, u2, u2, u2, b1))(u1[:], u4, u4)', - nopython=True, nogil=True) - def unpack_payload(buf, offset, length): - crc_fail = False - crc = 0 - payload_len = 0 - msg_type = 0 - sender = 0 - offset_start = offset - pkt_len = 0 - - preamble_len = 1 - if length < preamble_len: - return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) - - preamble, offset, length = get_u8(buf, offset, length) - if preamble != SBP_PREAMBLE: - return (preamble_len, payload_len, msg_type, sender, crc, crc_fail) - - header_len = 5 - if length < header_len: - return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) - - typ, offset, length = get_u16(buf, offset, length) - sender, offset, length = get_u16(buf, offset, length) - payload_len, offset, length = get_u8(buf, offset, length) - - if length < payload_len: - return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) - - # Consume payload - offset += payload_len - length -= payload_len - - crc_len = 2 - if length < crc_len: - return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) - - msg_type = typ - - crc, offset, length = get_u16(buf, offset, length) - buf_start = offset_start + 1 - buf_end = offset_start + 1 + (preamble_len + header_len - 1) + payload_len - - calc_crc = crc16jit(buf, buf_start, 0, buf_end - buf_start) - if calc_crc != crc: - crc_fail = True - - pkt_len = preamble_len + header_len + payload_len + crc_len - return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) - @classmethod def parse_members(cls, buf, offset, length): raise NotImplementedError() diff --git a/python/sbp/jit/parse.py b/python/sbp/jit/parse.py new file mode 100644 index 0000000000..34917b0a11 --- /dev/null +++ b/python/sbp/jit/parse.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python + +# Copyright (C) 2019 Swift Navigation Inc. +# Contact: Swift Navigation +# +# This source is subject to the license found in the file 'LICENSE' which must +# be be distributed together with this source. All other rights reserved. +# +# THIS CODE AND INFORMATION IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND, +# EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR PURPOSE. + +import glob +import ntpath +import os +import shutil +import importlib +import time + +import numpy as np +import numba as nb + +from numba.pycc import CC + +from sbp.constants import SENDER_ID as _SENDER_ID +from sbp.constants import SBP_PREAMBLE as _SBP_PREAMBLE +from sbp.constants import crc16_tab + +from distutils.ccompiler import CCompiler +from numpy.distutils.misc_util import get_num_build_jobs +from numpy.distutils.ccompiler import _global_lock, _processing_files +from numpy.distutils import log + +import sys +import types + +# monkeypatch numpy.distutils.ccompiler. Parallel build hangs with py27 +def replace_method(klass, method_name, func): + if sys.version_info[0] < 3: + m = types.MethodType(func, None, klass) + setattr(klass, method_name, m) + +# Original https://github.com/numpy/numpy/blob/v1.16.2/numpy/distutils/ccompiler.py#L223 +def CCompiler_compile(self, sources, output_dir=None, macros=None, + include_dirs=None, debug=0, extra_preargs=None, + extra_postargs=None, depends=None): + global _job_semaphore + + if not sources: + return [] + + ccomp = self.compiler_so + display = "C compiler: %s\n" % (' '.join(ccomp),) + log.info(display) + macros, objects, extra_postargs, pp_opts, build = \ + self._setup_compile(output_dir, macros, include_dirs, sources, + depends, extra_postargs) + cc_args = self._get_cc_args(pp_opts, debug, extra_preargs) + #cc_args += ['-g'] + display = "compile options: '%s'" % (' '.join(cc_args)) + if extra_postargs: + display += "\nextra options: '%s'" % (' '.join(extra_postargs)) + log.info(display) + + def single_compile(args): + obj, (src, ext) = args + + # check if we are currently already processing the same object + # happens when using the same source in multiple extensions + while True: + # need explicit lock as there is no atomic check and add with GIL + with _global_lock: + # file not being worked on, start working + if obj not in _processing_files: + _processing_files.add(obj) + break + # wait for the processing to end + time.sleep(0.1) + + try: + self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts) + finally: + # register being done processing + with _global_lock: + _processing_files.remove(obj) + + for o in build.items(): + single_compile(o) + + # Return *all* object filenames, not just the ones we just built. + return objects + +replace_method(CCompiler, 'compile', CCompiler_compile) +# monkeypatch end + +module_name = "parse_jit_py{}".format(str(sys.version_info[0]) + str(sys.version_info[1])) +cc = CC(module_name) +cc.verbose = True + +np_crc16_tab = np.array(crc16_tab, dtype=np.uint16) + +SENDER_ID = _SENDER_ID +SBP_PREAMBLE = _SBP_PREAMBLE + + +@nb.jit('u2(u1[:], u4, u2, u4)', nopython=True, nogil=True) +@cc.export('crc16jit', 'u2(u1[:], u4, u2, u4)') +def crc16jit(buf, offset, crc, length): + """CRC16 implementation acording to CCITT standards.""" + for index in range(offset, offset + length): + data = buf[index] + lookup = np_crc16_tab[((nb.u2(crc) >> 8) & nb.u2(0xFF)) ^ (data & nb.u2(0xFF))] + crc = ((nb.u2(crc) << nb.u2(8)) & nb.u2(0xFFFF)) ^ lookup + crc = nb.u2(crc) & nb.u2(0xFFFF) + return crc + + +@nb.jit('Tuple((u1,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) +@cc.export('get_u8', 'Tuple((u1,u4,u4))(u1[:],u4,u4)') +def get_u8(buf, offset, length): + if length < 1: + return (0, offset, length) + return buf[offset], offset + 1, length - 1 + + +@nb.jit('Tuple((u2,u4,u4))(u1[:],u4,u4)', nopython=True, nogil=True) +@cc.export('get_u16', 'Tuple((u2,u4,u4))(u1[:],u4,u4)') +def get_u16(buf, offset, length): + if length < 2: + return (0, offset, length) + msb = nb.u2(buf[offset + 1]) << 8 + lsb = nb.u2(buf[offset + 0]) << 0 + return msb | lsb, offset + 2, length - 2 + + +@cc.export('get_u32', 'Tuple((u4,u4,u4))(u1[:],u4,u4)') +def get_u32(buf, offset, length): + if length < 4: + return (0, offset, length) + a = nb.u4(buf[offset + 3]) << 24 + b = nb.u4(buf[offset + 2]) << 16 + c = nb.u4(buf[offset + 1]) << 8 + d = nb.u4(buf[offset + 0]) << 0 + return a | b | c | d, offset + 4, length - 4 + + +@cc.export('get_u64', 'Tuple((u8,u4,u4))(u1[:],u4,u4)') +def get_u64(buf, offset, length): + if length < 8: + return (0, offset, length) + a = nb.u8(buf[offset + 7]) << 54 + b = nb.u8(buf[offset + 6]) << 48 + c = nb.u8(buf[offset + 5]) << 40 + d = nb.u8(buf[offset + 4]) << 32 + e = nb.u8(buf[offset + 3]) << 24 + f = nb.u8(buf[offset + 2]) << 16 + g = nb.u8(buf[offset + 1]) << 8 + h = nb.u8(buf[offset + 0]) << 0 + return a | b | c | d | e | f | g | h, offset + 8, length - 8 + + +@cc.export('get_s8', 'Tuple((i1,u4,u4))(u1[:],u4,u4)') +def get_s8(buf, offset, length): + if length < 1: + return (0, offset, length) + return buf[offset], offset + 1, length - 1 + + +@cc.export('get_s16', 'Tuple((i2,u4,u4))(u1[:],u4,u4)') +def get_s16(buf, offset, length): + if length < 2: + return (0, offset, length) + msb = nb.i2(buf[offset + 1]) << 8 + lsb = nb.i2(buf[offset + 0]) << 0 + return msb | lsb, offset + 2, length - 2 + + +@cc.export('get_s32', 'Tuple((i4,u4,u4))(u1[:],u4,u4)') +def get_s32(buf, offset, length): + if length < 4: + return (0, offset, length) + a = nb.i4(buf[offset + 3]) << 24 + b = nb.i4(buf[offset + 2]) << 16 + c = nb.i4(buf[offset + 1]) << 8 + d = nb.i4(buf[offset + 0]) << 0 + return a | b | c | d, offset + 4, length - 4 + + +@cc.export('get_s64', 'Tuple((i8,u4,u4))(u1[:],u4,u4)') +def get_s64(buf, offset, length): + if length < 8: + return (0, offset, length) + a = nb.i8(buf[offset + 7]) << 54 + b = nb.i8(buf[offset + 6]) << 48 + c = nb.i8(buf[offset + 5]) << 40 + d = nb.i8(buf[offset + 4]) << 32 + e = nb.i8(buf[offset + 3]) << 24 + f = nb.i8(buf[offset + 2]) << 16 + g = nb.i8(buf[offset + 1]) << 8 + h = nb.i8(buf[offset + 0]) << 0 + return a | b | c | d | e | f | g | h, offset + 8, length - 8 + + +@cc.export('_get_string', 'Tuple((u1[:],u4,u4))(u1[:],u4,u4,b1)') +def _get_string(buf_in, offset, length, check_null): + buf_out = np.zeros(256, dtype=np.uint8) + i = nb.u4(0) + null_term = False + while i < length: + if check_null and buf_in[offset + i] == 0: + null_term = True + break + buf_out[i] = buf_in[offset + i] + i = nb.u4(i + nb.u4(1)) + if null_term: + return buf_out[:i], offset + i + 1, i + 1 + else: + return buf_out[:i], offset + i, i + + +@cc.export('unpack_payload', 'Tuple((u4, u2, u2, u2, u2, b1))(u1[:], u4, u4)') +def unpack_payload(buf, offset, length): + crc_fail = False + crc = 0 + payload_len = 0 + msg_type = 0 + sender = 0 + offset_start = offset + pkt_len = 0 + + preamble_len = 1 + if length < preamble_len: + return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) + + preamble, offset, length = get_u8(buf, offset, length) + if preamble != SBP_PREAMBLE: + return (preamble_len, payload_len, msg_type, sender, crc, crc_fail) + + header_len = 5 + if length < header_len: + return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) + + typ, offset, length = get_u16(buf, offset, length) + sender, offset, length = get_u16(buf, offset, length) + payload_len, offset, length = get_u8(buf, offset, length) + + if length < payload_len: + return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) + + # Consume payload + offset += payload_len + length -= payload_len + + crc_len = 2 + if length < crc_len: + return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) + + msg_type = typ + + crc, offset, length = get_u16(buf, offset, length) + buf_start = offset_start + 1 + buf_end = offset_start + 1 + (preamble_len + header_len - 1) + payload_len + + calc_crc = crc16jit(buf, buf_start, 0, buf_end - buf_start) + if calc_crc != crc: + crc_fail = True + + pkt_len = preamble_len + header_len + payload_len + crc_len + return (pkt_len, payload_len, msg_type, sender, crc, crc_fail) + + +def compile(): + cc.compile() + + # Move deliverables to same dir as the script + dest_dir = os.path.dirname(os.path.realpath(__file__)) + for f in glob.glob(os.path.join(os.getcwd(), module_name + '.*')): + shutil.move(f, os.path.join(dest_dir, ntpath.basename(f))) + + +if __name__ == "__main__": # not when running with setuptools + compile() diff --git a/python/sbp/jit/parse_float.py b/python/sbp/jit/parse_float.py deleted file mode 100644 index c011031265..0000000000 --- a/python/sbp/jit/parse_float.py +++ /dev/null @@ -1,49 +0,0 @@ -import cffi - -import glob -import ntpath -import os -import shutil - -ffi = cffi.FFI() -ffi.cdef(""" -float get_f32(unsigned char a, unsigned char b, unsigned char c, unsigned char d); -double get_f64(unsigned char a, unsigned char b, unsigned char c, unsigned char d, - unsigned char e, unsigned char f, unsigned char g, unsigned char h); -""") - -source = """ -float get_f32(unsigned char a, unsigned char b, unsigned char c, unsigned char d) { - union { unsigned char buf[4]; float f; } u; - u.buf[0] = a; - u.buf[1] = b; - u.buf[2] = c; - u.buf[3] = d; - return u.f; -} - -double get_f64(unsigned char a, unsigned char b, unsigned char c, unsigned char d, - unsigned char e, unsigned char f, unsigned char g, unsigned char h) { - union { unsigned char buf[8]; double d; } u; - u.buf[0] = a; - u.buf[1] = b; - u.buf[2] = c; - u.buf[3] = d; - u.buf[4] = e; - u.buf[5] = f; - u.buf[6] = g; - u.buf[7] = h; - return u.d; -} -""" - -module_name = "parse_float_c" - -ffi.set_source(module_name=module_name, source=source) - -ffi.compile() - -# Move deliverables to same dir as the script -dest_dir = os.path.dirname(os.path.realpath(__file__)) -for f in glob.glob(os.path.join(os.getcwd(), module_name + '.*')): - shutil.move(f, os.path.join(dest_dir, ntpath.basename(f))) diff --git a/python/sbp/msg.py b/python/sbp/msg.py index 748424edfe..b6e1022b5d 100755 --- a/python/sbp/msg.py +++ b/python/sbp/msg.py @@ -11,80 +11,55 @@ import base64 import copy +import importlib import json import struct import construct +from sbp.constants import SENDER_ID as _SENDER_ID +from sbp.constants import SBP_PREAMBLE as _SBP_PREAMBLE +from sbp.constants import crc16_tab + import numba as nb import numpy as np -SBP_PREAMBLE = 0x55 - -# Default sender ID. Intended for messages sent from the host to the -# device. -SENDER_ID = 0x42 - -_crc16_tab = [ - 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, - 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, - 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, - 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, - 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, - 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, - 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, - 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, - 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, - 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, - 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, - 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, - 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, - 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, - 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, - 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, - 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, - 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, - 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, - 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, - 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, - 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, - 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, - 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, - 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, - 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, - 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, - 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, - 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, - 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, - 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, - 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0] - -crc16_tab = np.array(_crc16_tab, dtype=np.uint16) - - -@nb.jit('u2(u1[:], u4, u2, u4)', nopython=True, nogil=True) -def crc16jit(buf, offset, crc, length): - """CRC16 implementation acording to CCITT standards.""" - for index in range(offset, offset + length): - data = buf[index] - lookup = crc16_tab[((nb.u2(crc) >> 8) & nb.u2(0xFF)) ^ (data & nb.u2(0xFF))] - crc = ((nb.u2(crc) << nb.u2(8)) & nb.u2(0xFFFF)) ^ lookup - crc = nb.u2(crc) & nb.u2(0xFFFF) - return crc +from pkgutil import iter_modules + +import sys + +parse_jit_name = "parse_jit_py{}".format(str(sys.version_info[0]) + str(sys.version_info[1])) + +if parse_jit_name in (name for loader, name, ispkg in iter_modules()): + # found in sys.path + parse_jit = importlib.import_module(parse_jit_name) +elif parse_jit_name in (name for loader, name, ispkg in iter_modules(['sbp/jit'])): + # found in sbp.jit + parse_jit = importlib.import_module('sbp.jit.' + parse_jit_name) +else: + # not found -> compile + from sbp.jit import parse + parse.compile() + parse_jit = importlib.import_module('sbp.jit.' + parse_jit_name) + + +np_crc16_tab = np.array(crc16_tab, dtype=np.uint16) +SENDER_ID = _SENDER_ID +SBP_PREAMBLE = _SBP_PREAMBLE crc_buffer = np.zeros(512, dtype=np.uint8) def crc16(s, crc=0, buf=crc_buffer): crc_buffer[:len(s)] = bytearray(s) - return crc16jit(crc_buffer, 0, crc, len(s)) + return parse_jit.crc16jit(crc_buffer, 0, crc, len(s)) def crc16_nojit(s, crc=0): """CRC16 implementation acording to CCITT standards.""" for ch in bytearray(s): # bytearray's elements are integers in both python 2 and 3 - crc = ((crc << 8) & 0xFFFF) ^ _crc16_tab[((crc >> 8) & 0xFF) ^ (ch & 0xFF)] + crc = ((crc << 8) & 0xFFFF) ^ np_crc16_tab[((crc >> 8) & 0xFF) ^ (ch & 0xFF)] crc &= 0xFFFF return crc @@ -185,7 +160,7 @@ def _get_framed(self, buf, offset, insert_payload): crc_offset = header_offset + self.length preamble_bytes = 1 crc_over_len = self._header_len + self.length - preamble_bytes - self.crc = crc16jit(buf, offset+1, 0, crc_over_len) + self.crc = parse_jit.crc16jit(buf, offset+1, 0, crc_over_len) struct.pack_into(self._crc_fmt, buf, crc_offset, self.crc) length = preamble_bytes + crc_over_len + self._crc_len return length diff --git a/python/setup.py b/python/setup.py index c3736e790a..55df2e039d 100755 --- a/python/setup.py +++ b/python/setup.py @@ -7,6 +7,7 @@ # https://github.com/swift-nav/traitsui/blob/swift-2019.01/setup.py # +import warnings from setuptools import setup import re @@ -30,6 +31,7 @@ PACKAGES = [ 'sbp', + 'sbp.jit', 'sbp.client', 'sbp.client.drivers', 'sbp.client.loggers', @@ -42,15 +44,16 @@ 'win32', ] +setup_py_dir = os.path.dirname(os.path.abspath(__file__)) def _read_release_version(): - this_dir = os.path.dirname(__file__) - relver_path = os.path.join(this_dir, 'sbp/RELEASE-VERSION') + relver_path = os.path.join(setup_py_dir, 'sbp/RELEASE-VERSION') try: with open(relver_path, "r") as f: version = f.readlines()[0] return version.strip() - except IOError: + except IOError as ex: + warnings.warn("Error reading version: {}".format(ex)) return "0.0.0" @@ -110,10 +113,8 @@ def _minimal_ext_cmd(cmd): def write_version_py(filename=VERSION_PY_PATH): - filedir = os.path.abspath(os.path.dirname(__file__)) - fullversion = VERSION - if os.path.exists(os.path.join(filedir, '..', '.git')): + if os.path.exists(os.path.join(setup_py_dir, '..', '.git')): git_rev, dev_num = git_version() elif os.path.exists(VERSION_PY_PATH): # must be a source distribution, use existing version file @@ -144,7 +145,8 @@ def write_version_py(filename=VERSION_PY_PATH): # fullversion += '.dev{0}+g{1}'.format(dev_num, git_rev) - filename_fullpath = os.path.join(filedir, filename) + filename_fullpath = os.path.join(setup_py_dir, filename) + print(filename_fullpath) with open(filename_fullpath, "wt") as fp: fp.write(VERSION_PY_TEMPLATE.format(version=VERSION, @@ -155,22 +157,33 @@ def write_version_py(filename=VERSION_PY_PATH): if __name__ == "__main__": - filedir = os.path.abspath(os.path.dirname(__file__)) - - with open(os.path.join(filedir, 'README.rst')) as f: + with open(os.path.join(setup_py_dir, 'README.rst')) as f: readme = f.read() - with open(os.path.join(filedir, 'requirements.txt')) as f: - INSTALL_REQUIRES = [i.strip() for i in f.readlines()] + INSTALL_REQUIRES = [] + with open(os.path.join(setup_py_dir, 'requirements.txt')) as f: + INSTALL_REQUIRES += [i.strip() for i in f.readlines()] - with open(os.path.join(filedir, 'test_requirements.txt')) as f: + with open(os.path.join(setup_py_dir, 'test_requirements.txt')) as f: TEST_REQUIRES = [i.strip() for i in f.readlines()] + with open(os.path.join(setup_py_dir, 'setup_requirements.txt')) as f: + SETUP_REQUIRES = [i.strip() for i in f.readlines() + if 'setuptools' not in i] + write_version_py() - from sbp import __version__ + + from sbp import __version__ as sbp_version + print("Building/installing libsbp version {} (read version: {})".format(sbp_version, VERSION)) + + ext_modules = None + if not os.environ.get('LIBSBP_BUILD_ANY', None): + from sbp.jit.parse import cc + ext_modules = [cc.distutils_extension()] + INSTALL_REQUIRES.extend(SETUP_REQUIRES) setup(name='sbp', - version=__version__, + version=sbp_version, description='Python bindings for Swift Binary Protocol', long_description=readme, author='Swift Navigation', @@ -182,4 +195,5 @@ def write_version_py(filename=VERSION_PY_PATH): install_requires=INSTALL_REQUIRES, tests_require=TEST_REQUIRES, use_2to3=False, - zip_safe=False) + zip_safe=False, + ext_modules=ext_modules) diff --git a/python/setup_requirements.txt b/python/setup_requirements.txt new file mode 100644 index 0000000000..4237b269a6 --- /dev/null +++ b/python/setup_requirements.txt @@ -0,0 +1,3 @@ +setuptools==41.0.1 +numba==0.41.0 +llvmlite==0.26.0 diff --git a/python/tests/sbp/test_numba.py b/python/tests/sbp/test_numba.py index ab2ba285c1..3d3705f8e0 100644 --- a/python/tests/sbp/test_numba.py +++ b/python/tests/sbp/test_numba.py @@ -3,7 +3,7 @@ from sbp.file_io import MsgFileioWriteReq -from sbp.jit.msg import SBP +from sbp.jit.msg import unpack_payload from sbp.jit.msg import get_string from sbp.jit.msg import get_fixed_string @@ -74,7 +74,7 @@ def test_parse(): assert len(buf) > 0 - pkt_len, payload_len, msg_type, sender, crc, crc_fail = SBP.unpack_payload(buf, 0, len(buf)) + pkt_len, payload_len, msg_type, sender, crc, crc_fail = unpack_payload(buf, 0, len(buf)) assert not crc_fail m = dispatch(msg_type)(msg_type) diff --git a/python/tox.ini b/python/tox.ini index bb47b36b96..61091909f2 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -1,12 +1,17 @@ [tox] envlist = py27, py35, py37 minversion = 1.7.2 +# Same as setup_requirements.txt as tox builds sdist package which it then +# installs into the testenvs +requires = setuptools==41.0.1 + numba==0.41.0 + llvmlite==0.26.0 [testenv] -deps = -r{toxinidir}/requirements.txt +deps = -r{toxinidir}/setup_requirements.txt + -r{toxinidir}/requirements.txt -r{toxinidir}/test_requirements.txt commands = - python {toxinidir}/sbp/jit/parse_float.py py.test -v tests/ py35,py37: {toxinidir}/../test_data/format-test.sh {posargs} {toxinidir}/../test_data/benchmark.sh {posargs} diff --git a/test_data/benchmark.sh b/test_data/benchmark.sh index f74b9e3625..d9b9aa863f 100755 --- a/test_data/benchmark.sh +++ b/test_data/benchmark.sh @@ -18,7 +18,7 @@ echo "Python" $time_py time_hs=$(TIMEFORMAT="%R"; { time $1/sbp2json < $TESTDATA_ROOT/long.sbp > $TESTDATA_ROOT/long_hask.json; } 2>&1) echo "Haskell" $time_hs -threshold=1.8 +threshold=1.4 perf_diff=$(echo "$time_py / $time_hs" | bc -l) if (( $(echo "$perf_diff > $threshold" | bc -l) )); then