Purge llvmlite and numba dependencies from generated Python code [ESD-1396]

HOWTO.md

@@ -188,7 +188,7 @@ The Linux x86 build of libsbp can be done throuch docker via the "manylinux"
 project by running the following set of commands:
 ```
 docker build -f python/Dockerfile.x86_64 -t libsbp-amd64 .
-docker run -v linux-amd64-root:/root -v $PWD:/work --rm -it libsbp-amd64 /bin/bash
+docker run -v libsbp-amd64-root:/root -v $PWD:/work --rm -it libsbp-amd64 /bin/bash
 cd /work
 make dist-python PYPI_USERNAME=swiftnav PYPI_PASSWORD=...
 ```
@@ -197,7 +197,7 @@ The Linux ARM build of libsbp can be done through docker via the following set
 of commands:
 ```
 docker build -f python/Dockerfile.arm -t libsbp-arm .
-docker run -v linux-arm-root:/root -v $PWD:/work --rm -it libsbp-arm /bin/bash
+docker run -v libsbp-arm-root:/root -v $PWD:/work --rm -it swiftnav/piksi-tools-stretch-arm:2019.06.25 libsbp-arm /bin/bash
 cd /work
 make dist-python PYPI_USERNAME=swiftnav PYPI_PASSWORD=...
 ```

c/include/libsbp/version.h

@@ -25,7 +25,7 @@
 /** Protocol minor version. */
 #define SBP_MINOR_VERSION 6
 /** Protocol patch version. */
-#define SBP_PATCH_VERSION 3
+#define SBP_PATCH_VERSION 4
 
 /** \} */
 

generator/requirements.txt

@@ -1,5 +1,5 @@
 Jinja2
-ruamel.yaml
+pyyaml
 coverage
 docutils
 flake8

generator/sbpg/specs/yaml2.py

@@ -22,8 +22,7 @@
 import sys
 import inspect
 
-from ruamel.yaml import YAML
-yaml = YAML(typ='safe')
+import yaml
 
 ##############################################################################
 #

generator/sbpg/targets/pythonNG.py

@@ -22,17 +22,17 @@
 
 TEMPLATE_NAME = "sbp_numba.py.j2"
 
-NUMBA_TYPE = {
-  'u8': 'u1',
-  'u16': 'u2',
-  'u32': 'u4',
-  'u64': 'u8',
-  's8': 'i1',
-  's16': 'i2',
-  's32': 'i4',
-  's64': 'i8',
-  'float': 'f4',
-  'double': 'f8',
+NUMPY_TYPE = {
+  'u8': 'uint8',
+  'u16': 'uint16',
+  'u32': 'uint32',
+  'u64': 'uint64',
+  's8': 'int8',
+  's16': 'int16',
+  's32': 'int32',
+  's64': 'int64',
+  'float': 'float32',
+  'double': 'float64',
 }
 
 NUMBA_GET_FN = {
@@ -81,7 +81,7 @@ def is_array():
 
 def numba_type(f):
   if f.type_id == 'float':
-    return 'judicious_round(nb.' + NUMBA_TYPE[f.type_id] + \
+    return 'judicious_round(np.' + NUMPY_TYPE[f.type_id] + \
       '(__' + f.identifier + ')) if SBP.judicious_rounding else __' + f.identifier
   else:
     return '__' + f.identifier
@@ -131,7 +131,7 @@ def numba_format(f):
       el_size = NUMBA_TY_BYTES[t]
       if f.options['fill'].value == 'float':
         return "get_fixed_array(%s, %d, %d, %s if SBP.judicious_rounding else None)" \
-          % (fill_func, count, el_size, 'nb.f4')
+          % (fill_func, count, el_size, 'np.float32')
       else:
         return "get_fixed_array(%s, %d, %d)" % (fill_func, count, el_size)
     else:

generator/sbpg/targets/resources/sbp_numba.py.j2

@@ -16,7 +16,7 @@
 
 import json
 
-import numba as nb
+import numpy as np
 
 from sbp.jit.msg import SBP, SENDER_ID
 from sbp.jit.msg import get_u8, get_u16, get_u32, get_u64

generator/setup.py

@@ -20,7 +20,6 @@
       url='https://github.com/swift-nav/libsbp/generator',
       install_requires=[
           'Jinja2',
-          'ruamel.yaml',
           'voluptuous'
       ],
       keywords='',

haskell/sbp.cabal

@@ -1,5 +1,5 @@
 name:                  sbp
-version:               2.6.3
+version:               2.6.4
 synopsis:              SwiftNav's SBP Library
 homepage:              https://github.com/swift-nav/libsbp
 license:               LGPL-3

package.json

@@ -1,6 +1,6 @@
 {
   "name": "sbp",
-  "version": "2.6.3",
+  "version": "2.6.4",
   "description": "libsbp bindings for JavaScript. More information here: http://swift-nav.github.io/libsbp/",
   "files": [
     "javascript/*",

python/Dockerfile.arm

@@ -1,15 +1,171 @@
-FROM balenalib/armv7hf-debian:sid-build
+FROM swiftnav/llvm-arm-stretch:2019.06.26
+
+ARG JOBS=8
 
 RUN [ "cross-build-start" ]
 
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /work
+
 RUN \
-     echo Setting up ARM build environment... \
+     echo '>>> Setting up ARM build environment...' \
   && apt-get update \
-  && apt-get install wget bzip2 build-essential llvm-6.0-dev python3 \
-  && update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-6.0 1 \
-  && wget -O /tmp/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-armv7l.sh \
-  && bash /tmp/miniconda.sh -b \
-  && conda install --yes pip \
-  && rm /tmp/miniconda.sh
-
-ENV PATH=/root/miniconda3/bin:$PATH
+  && apt-get install -y wget bzip2 build-essential python3 \
+  && apt-get install -y build-essential tk-dev libncurses5-dev libncursesw5-dev \
+                        libreadline6-dev libdb5.3-dev libgdbm-dev libsqlite3-dev \
+                        libssl-dev libbz2-dev libexpat1-dev liblzma-dev zlib1g-dev \
+                        libffi-dev cmake ccache binutils-dev ninja-build \
+                        gperf gawk flex bison ncurses-dev m4 patchelf \
+                        texinfo help2man libpthread-stubs0-dev libtinfo-dev \
+                        libatlas-base-dev libsqlite3-dev tk-dev libgdbm-dev \
+                        libc6-dev
+
+RUN \
+     echo '>>> Downlading get-pip.py...' \
+  && wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
+
+ENV OPENSSL_VER=1.0.2s
+ENV OPENSSL_PREFIX=/opt/openssl-${OPENSSL_VER}
+ENV OPENSSL_URL=https://www.openssl.org/source/openssl-${OPENSSL_VER}.tar.gz
+
+RUN \
+     echo ">>> Building OpenSSL ${OPENSSL_VER}..." \
+  && wget -O /tmp/openssl.tgz ${OPENSSL_URL} \
+  && mkdir /tmp/openssl-build-${OPENSSL_VER} \
+  && cd /tmp/openssl-build-${OPENSSL_VER} \
+  && tar -xzf /tmp/openssl.tgz \
+  && cd openssl-${OPENSSL_VER} \
+  && ./config shared --prefix=${OPENSSL_PREFIX} \
+  && make -j${JOBS} \
+  && make -j${JOBS} install \
+  && cd /work
+
+ENV PY34=3.4.10
+ENV PY34_URL=https://www.python.org/ftp/python/${PY34}/Python-${PY34}.tar.xz
+
+RUN \
+     echo '>>> Building Python 3.4...' \
+  && wget -O /tmp/python${PY34}.txz ${PY34_URL} \
+  && mkdir /tmp/python${PY34}-build && cd /tmp/python${PY34}-build \
+  && tar -xJf /tmp/python${PY34}.txz \
+  && cd Python-${PY34} \
+  && ./configure \
+      LDFLAGS="-L${OPENSSL_PREFIX}/lib -Wl,-rpath=${OPENSSL_PREFIX}/lib" \
+      CPPFLAGS="-I${OPENSSL_PREFIX}/include" \
+  \
+  && make -j${JOBS} \
+  && make altinstall \
+  && cd /work \
+  \
+  && /usr/local/bin/python3.4 /tmp/get-pip.py \
+  && /usr/local/bin/python3.4 -m pip install --upgrade pip \
+  && /usr/local/bin/python3.4 -m pip install wheel setuptools
+
+ENV PY27=2.7.16
+ENV PY27_URL=https://www.python.org/ftp/python/${PY27}/Python-${PY27}.tar.xz
+
+RUN \
+     echo '>>> Downlading Python 2.7...' \
+  && wget -O /tmp/python${PY27}.txz ${PY27_URL}
+
+RUN \
+     echo '>>> Building Python 2.7...' \
+  \
+  && mkdir /tmp/python${PY27}-build && cd /tmp/python${PY27}-build \
+  && tar -xJf /tmp/python${PY27}.txz \
+  && cd Python-${PY27} \
+  && ./configure \
+  && make -j${JOBS} \
+  && make altinstall \
+  && cd /work \
+  \
+  && /usr/local/bin/python2.7 /tmp/get-pip.py \
+  && /usr/local/bin/python2.7 -m pip install --upgrade pip \
+  && /usr/local/bin/python2.7 -m pip install wheel setuptools
+
+ENV PY27MU=2.7.16-cp27mu
+ENV CP27MU_PREFIX=/opt/cp27mu
+
+RUN \
+     echo '>>> Building Python 2.7 (cp27mu)...' \
+  \
+  && mkdir /tmp/python${PY27MU}-build && cd /tmp/python${PY27MU}-build \
+  && tar -xJf /tmp/python${PY27}.txz \
+  && cd Python-${PY27} \
+  && ./configure \
+      --enable-unicode=ucs4 \
+      --prefix=${CP27MU_PREFIX} \
+  && make -j${JOBS} \
+  && make altinstall \
+  && cd /work \
+  \
+  && ${CP27MU_PREFIX}/bin/python2.7 /tmp/get-pip.py \
+  && ${CP27MU_PREFIX}/bin/python2.7 -m pip install --upgrade pip \
+  && ${CP27MU_PREFIX}/bin/python2.7 -m pip install wheel setuptools
+
+ENV PY35=3.5.7
+ENV PY35_URL=https://www.python.org/ftp/python/${PY35}/Python-${PY35}.tar.xz
+
+RUN \
+     echo '>>> Building Python 3.5...' \
+  && wget -O /tmp/python${PY35}.txz ${PY35_URL} \
+  && mkdir /tmp/python${PY35}-build \
+  && cd /tmp/python${PY35}-build \
+  && tar -xJf /tmp/python${PY35}.txz \
+  && cd Python-${PY35} \
+  && ./configure \
+  && make -j${JOBS} \
+  && make altinstall \
+  && cd /work \
+  \
+  && /usr/local/bin/python3.5 -m pip install --upgrade pip \
+  && /usr/local/bin/python3.5 -m pip install wheel setuptools
+
+ENV PY36=3.6.8
+ENV PY36_URL=https://www.python.org/ftp/python/${PY36}/Python-${PY36}.tar.xz
+
+RUN \
+     echo '>>> Building Python 3.6...' \
+  && wget -O /tmp/python${PY36}.txz ${PY36_URL} \
+  && mkdir /tmp/python${PY36}-build && cd /tmp/python${PY36}-build \
+  && tar -xJf /tmp/python${PY36}.txz \
+  && cd Python-${PY36} \
+  && ./configure \
+  && make -j${JOBS} \
+  && make altinstall \
+  && cd /work \
+  \
+  && /usr/local/bin/python3.6 -m pip install --upgrade pip \
+  && /usr/local/bin/python3.6 -m pip install wheel setuptools
+
+ENV PY37=3.7.3
+ENV PY37_URL=https://www.python.org/ftp/python/${PY37}/Python-${PY37}.tar.xz
+
+RUN \
+     echo '>>> Building Python 3.7...' \
+  && wget -O /tmp/python${PY37}.txz ${PY37_URL} \
+  && mkdir /tmp/python${PY37}-build && cd /tmp/python${PY37}-build \
+  && tar -xJf /tmp/python${PY37}.txz \
+  && cd Python-${PY37} \
+  && ./configure \
+  && make -j${JOBS} \
+  && make altinstall \
+  && cd /work \
+  && /usr/local/bin/python3.7 -m pip install --upgrade pip \
+  && /usr/local/bin/python3.7 -m pip install wheel setuptools
+
+RUN \
+     rm /tmp/python${PY27}.txz \
+  && rm -rf /tmp/python${PY27}-build \
+  && rm /tmp/python${PY34}.txz \
+  && rm -rf /tmp/python${PY34}-build \
+  && rm /tmp/python${PY35}.txz \
+  && rm -rf /tmp/python${PY35}-build \
+  && rm /tmp/python${PY36}.txz \
+  && rm -rf /tmp/python${PY36}-build \
+  && rm /tmp/python${PY37}.txz \
+  && rm -rf /tmp/python${PY37}-build \
+  && rm -rf /tmp/openssl.tgz \
+  && rm -rf /tmp/openssl-build-${OPENSSLVER} \
+  && rm /tmp/get-pip.py

python/Dockerfile.llvm_arm

@@ -0,0 +1,50 @@
+FROM swiftnav/llvm-arm:2019.06.26
+
+RUN [ "cross-build-start" ]
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /work
+
+RUN \
+     echo '>>> Setting up ARM build environment...' \
+  && apt-get update \
+  && apt-get install -y wget bzip2 build-essential python3 \
+  && apt-get install -y build-essential tk-dev libncurses5-dev libncursesw5-dev \
+                        libreadline6-dev libdb5.3-dev libgdbm-dev libsqlite3-dev \
+                        libssl-dev libbz2-dev libexpat1-dev liblzma-dev zlib1g-dev \
+                        libffi-dev cmake ccache binutils-dev ninja-build \
+                        gperf gawk flex bison ncurses-dev m4 patchelf \
+                        texinfo help2man libpthread-stubs0-dev libtinfo-dev \
+                        libatlas-base-dev libsqlite3-dev tk-dev libgdbm-dev \
+                        libc6-dev
+
+ENV LLVM_BRANCH=release_60
+ENV LLVM_URL=https://github.com/llvm-mirror/llvm.git
+ENV CLANG_URL=https://github.com/llvm-mirror/clang.git
+
+RUN \
+     echo '>>> Building LLVM 6.0...' \
+  && git clone --depth=1 --single-branch -b ${LLVM_BRANCH} ${LLVM_URL} /tmp/llvm \
+  && git clone --depth=1 --single-branch -b ${LLVM_BRANCH} ${CLANG_URL} /tmp/llvm/tools/clang \
+  && mkdir -p /tmp/llvm/build \
+  && cd /tmp/llvm/build \
+  && cmake -G Ninja /tmp/llvm \
+      -DCMAKE_INSTALL_PREFIX=/usr/local \
+      -DLLVM_TARGETS_TO_BUILD=ARM \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_BINUTILS_INCDIR=/usr/include \
+      -DLLVM_INCLUDE_TESTS=OFF \
+      -DLLVM_ENABLE_TERMINFO=0 \
+      -DCMAKE_C_COMPILER=/usr/bin/cc \
+      -DCMAKE_CXX_COMPILER=/usr/bin/c++ \
+      -DCMAKE_ASM_COMPILER=/usr/bin/as \
+      -DCMAKE_CXX_FLAGS="-I/root/llvm/build/tools/clang/include -I/root/llvm/tools/clang/include -DENDIAN_LITTLE=1" \
+  && ninja -v \
+  && ninja install
+
+RUN \
+     echo '>>> Cleaning up...' \
+  && rm -rf /tmp/llvm
+
+RUN [ "cross-build-end" ]