rust: implement sbp2json, json2sbp, json2json [INFRA-125]

.gitignore

@@ -58,9 +58,6 @@ coverage.xml
 # Sphinx documentation
 docs/_build/
 
-# PyBuilder
-target/
-
 # LaTeX output
 sbp_out.*
 
@@ -74,5 +71,6 @@ sbp_out.*
 
 # rust
 Cargo.lock
+/target
 
 python/sbp/_version.py

.travis.yml

@@ -1,23 +1,32 @@
+# yamllint disable rule:line-length
+---
 sudo: required
 
+branches:
+  only:
+    - master
+
 dist: bionic
 language: generic
 
 addons:
   apt:
     packages:
-    - build-essential
-    - doxygen
-    - cmake
-    - lcov
-    - gradle
-    - libgmp-dev
+      - build-essential
+      - doxygen
+      - cmake
+      - lcov
+      - gradle
+      - libgmp-dev
 
 cache:
   apt: true
+  cargo: true
   directories:
-    - $HOME/.stack/
     - .stack-work/
+    - $HOME/.stack/
+    - $HOME/.cargo/
+    - $HOME/.docker-cache/
 
 matrix:
   include:
@@ -27,25 +36,17 @@ matrix:
       python: 3.7-dev
       before_install:
         - sudo apt-get -qq update
-        - sudo apt-get install python2.7 python2.7-dev
-
+        - sudo apt-get install gcc python2.7 python2.7-dev
         # https://linuxize.com/post/how-to-install-python-3-8-on-ubuntu-18-04/
         - sudo apt-get install software-properties-common
         - sudo add-apt-repository -y ppa:deadsnakes/ppa
         - sudo apt-get install python3.5 python3.5-dev
         - sudo apt-get install python3.8 python3.8-dev
-
         - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce
         - sudo pip install tox
-      script: |
-        pushd haskell
-        docker build -t sbp2json .
-        docker run --rm --name sbp2json sbp2json >sbp_linux_tools.tar
-        tar xf sbp_linux_tools.tar
-        gzip sbp_linux_tools.tar
-        ls -l
-        popd
-        make test-python
+      script:
+        - ./scripts/travis_python_haskell.bash
+
     - env:
         - TRAVIS_TARGET=default JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
       before_install:
@@ -80,7 +81,9 @@ matrix:
         - make test-c
       after_success:
         bash <(curl -s https://codecov.io/bash) -s c/build || echo "Codecov did not collect coverage reports";
+
     - env:
+        - TRAVIS_TARGET=rust
       before_install:
         - sudo apt-get -qq update
         - sudo apt-get install libudev-dev
@@ -90,11 +93,83 @@ matrix:
       script:
         - make test-rust
 
-deploy:
-  provider: releases
+    - env:
+        - TRAVIS_TARGET=benchmark
+        - VIA_DOCKER=y
+      language: python
+      python:
+        - "3.7"
+      before_install:
+        - source ./scripts/travis_benchmark_install.bash
+      script:
+        - ./scripts/travis_benchmark.bash
+
+    - env:
+        - TRAVIS_TARGET=osx
+      os: osx
+      osx_image: Xcode 8.3.3
+      language: rust
+      script: |
+        set -ex
+        cargo build --all --release
+        tar -C "target/release" -czf sbp_osx_tools.tar.gz \
+          sbp2json json2sbp json2json
+        VERSION="$(git describe --always --tags --dirty)"
+        BUILD_TRIPLET="$(cc -dumpmachine)"
+        mv sbp_osx_tools.tar.gz "sbp_tools-${VERSION}-${BUILD_TRIPLET}.tar.gz"
+        ls -l
+        set +ex
+      addons:
+        homebrew:
+          packages:
+            - cmake
+
+    - env:
+        - TRAVIS_TARGET=windows
+      os: windows
+      language: rust
+      script: |
+        cargo build --all --release
+        cd target/release
+        7z a -tzip ../../sbp_windows_tools.zip \
+          sbp2json.exe json2sbp.exe json2json.exe
+        cd ../..
+        VERSION="$(git describe --always --tags --dirty)"
+        BUILD_TRIPLET="$(clang -dumpmachine)"
+        mv sbp_windows_tools.zip "sbp_tools-${VERSION}-${BUILD_TRIPLET}.zip"
+        ls -l
+
+_github_api_key: &_github_api_key
   api_key:
     secure: "WDZRT7VevzQidPo8Mn1ozV6azwNFe7qGwKbLCb8rmKpPmiF+wcQ3KrhTnGj0EmqGqQ2q1c3UZzVCr+nLhLvIFLhkvNK42bs8Byc+HolKNX/diPAlB/mczGHI08rV4EJFmJL56vopdoafOAsqtZBnD4hZ8BDETNqMyJefo70pyY8="
-  file: haskell/sbp_linux_tools.tar.gz
-  skip_cleanup: true
-  on:
-    tags: true
+
+deploy:
+  - provider: releases
+    file: "haskell/sbp_tools_haskell-*.tar.gz"
+    skip_cleanup: true
+    "on":
+      tags: true
+      condition: "$TRAVIS_TARGET = python-haskell"
+    <<: *_github_api_key
+  - provider: releases
+    file: "rust/sbp_tools-*.tar.gz"
+    skip_cleanup: true
+    "on":
+      tags: true
+      condition: "$TRAVIS_TARGET = benchmark"
+    <<: *_github_api_key
+  - provider: releases
+    file: "sbp_tools-*.tar.gz"
+    skip_cleanup: true
+    "on":
+      tags: true
+      condition: "$TRAVIS_TARGET = osx"
+    <<: *_github_api_key
+  - provider: releases
+    file: "sbp_tools-*.zip"
+    skip_cleanup: true
+    "on":
+      tags: true
+      condition: "$TRAVIS_TARGET = windows"
+    <<: *_github_api_key
+...

Cargo.toml

@@ -0,0 +1,5 @@
+[workspace]
+members = [
+  "rust/sbp",
+  "rust/sbp2json"
+]

HOWTO.md

@@ -11,6 +11,8 @@ libsbp Development Procedures
   * [Building on Windows](#building-on-windows)
   * [Troubleshooting](#troubleshooting)
     + [Error: `!!! No Python wheel (.whl) file found...`](#error--no-python-wheel-whl-file-found)
+    + [Tox error: `ERROR: FAIL could not package project`](#tox-error-error-fail-could-not-package-project)
+    + [Tox error: `ERROR: cowardly refusing to delete envdir`](#tox-error-error-cowardly-refusing-to-delete-envdir)
 - [Contributions](#contributions)
 
 <!-- tocstop -->

Makefile

@@ -320,9 +320,9 @@ test-haskell:
 
 test-rust:
 	$(call announce-begin,"Running Rust tests")
-	cd $(SWIFTNAV_ROOT)/rust/sbp && cargo test --verbose
+	cargo test --verbose --all-targets
 	$(call announce-begin,"Building Rust examples")
-	cd $(SWIFTNAV_ROOT)/rust/sbp && cargo build --examples --verbose --all-features --all-targets
+	cargo build --examples --verbose --all-features --all-targets
 	$(call announce-end,"Finished running Rust tests")
 
 test-protobuf:
@@ -406,3 +406,6 @@ mapping:
     awk '{printf("%s  %5d  %s\n", $$4, $$4, $$2)}' |\
 		sed 's/:$$//' |\
     sort
+
+benchmark:
+	@PYTHONPATH=$(PWD)/test_data ./test_data/benchmark_main.py

README.md

@@ -2,34 +2,102 @@
 
 [![Build status][1]][2]
 
-The Swift Navigation Binary Protocol (SBP) is a fast, simple, and
-minimal binary protocol for communicating with Swift devices. It is
-the native binary protocol used by the
-[Piksi GPS receiver](http://swiftnav.com/piksi.html) to transmit
-solutions, observations, status and debugging messages, as well as
+<!-- toc -->
+
+- [Installing sbp2json, json2sbp, json2json and related tools](#installing-sbp2json-json2sbp-json2json-and-related-tools)
+- [Building / installing](#building--installing)
+  * [Using Docker](#using-docker)
+    + [Creating your own image](#creating-your-own-image)
+    + [Using the prebuilt one from dockerhub](#using-the-prebuilt-one-from-dockerhub)
+  * [Installing from package managers](#installing-from-package-managers)
+  * [Installing development Python versions](#installing-development-python-versions)
+  * [Adding development version as a pip dependency](#adding-development-version-as-a-pip-dependency)
+  * [Installing from source](#installing-from-source)
+- [SBP Protocol Specification](#sbp-protocol-specification)
+- [JSON Schema Definitions](#json-schema-definitions)
+- [LICENSE](#license)
+
+<!-- tocstop -->
+
+The Swift Navigation Binary Protocol (SBP) is a fast, simple, and minimal
+binary protocol for communicating with Swift devices. It is the native binary
+protocol used by the [Piksi GPS receiver](http://swiftnav.com/piksi.html) to
+transmit solutions, observations, status and debugging messages, as well as
 receive messages from the host operating system, such as differential
 corrections and the almanac.
 
-This project provides language-agnostic specification and
-documentation for messages used with SBP, a compiler for generating
-message bindings, and client libraries in a variety of languages. This
-repository is organized into the following directory structure:
+This project provides language-agnostic specification and documentation for
+messages used with SBP, a compiler for generating message bindings, and client
+libraries in a variety of languages. This repository is organized into the
+following directory structure:
 
-* [`docs`](https://github.com/swift-nav/libsbp/tree/HEAD/docs): Protocol documentation and message definitions.
-* [`spec`](https://github.com/swift-nav/libsbp/tree/HEAD/spec): Machine readable protocol specification in
+* [`docs`](./docs): Protocol documentation and message definitions.
+* [`spec`](./spec): Machine readable protocol specification in
   [YAML](http://en.wikipedia.org/wiki/YAML).
-* [`generator`](https://github.com/swift-nav/libsbp/tree/HEAD/generator): Simple, template-based generator for
+* [`generator`](./generator): Simple, template-based generator for
   different languages.
-* [`python`](https://github.com/swift-nav/libsbp/tree/HEAD/python): Python client and examples.
-* [`c`](https://github.com/swift-nav/libsbp/tree/HEAD/c): C client library and examples.
-* [`haskell`](https://github.com/swift-nav/libsbp/tree/HEAD/haskell): Haskell client and examples.
-* [`java`](https://github.com/swift-nav/libsbp/tree/HEAD/java): Java client library and examples.
-* [`javascript`](https://github.com/swift-nav/libsbp/tree/HEAD/javascript): JavaScript client library and examples.
-* [`rust`](https://github.com/swift-nav/libsbp/tree/HEAD/rust): Rust client library and examples.
-* [`sbpjson`](https://github.com/swift-nav/libsbp/tree/HEAD/sbpjson): Tools for parsing SBP-JSON.
+* [`python`](./python): Python client and examples.
+* [`c`](./c): C client library and examples.
+* [`haskell`](./haskell): Haskell client and examples.
+* [`java`](./java): Java client library and examples.
+* [`javascript`](./javascript): JavaScript client library and examples.
+* [`rust`](./rust): Rust client library and examples.
+* [`sbpjson`](./sbpjson): Tools for parsing SBP-JSON.
 
 Except for the `generator`, all of the above are generated and should not be modified directly.
 
+## Installing sbp2json, json2sbp, json2json and related tools
+
+This repository also provides the following utilities for comprehending and
+inspecting SBP data:
+
+- `sbp2json`: converts SBP binary into a JSON representation, in which field
+  names and values are expanded into JSON objects, common fields such as
+  "message type" and "payload" are included as well.
+
+- `json2sbp`: uses the "message type", "payload" and related fields from an SBP
+  JSON stream to reconstruct the binary representation.
+
+- `json2json`: some tools (notably the Swift GUI Console) produce abbreviated
+  JSON logs with only common fields such as "message type" and "payload", the
+  `json2json` tool expands these JSON objects to include fields specific the
+  individual message.
+
+To install a released version of these tools, visit the [releases
+page](https://github.com/swift-nav/libsbp/releases) and download an archive for
+your platform.
+
+To install from source, you can use Rust's cargo tool (first [install
+Rust](https://www.rust-lang.org/tools/install)), then run:
+
+```
+cargo install --git https://github.com/swift-nav/libsbp.git --bins
+```
+
+There's also a [Haskell version](./haskell) available which can be installed by
+running `stack install` in the [./haskell](./haskell) directory of a checkout
+of this repo (after [installing
+stack](https://docs.haskellstack.org/en/stable/README/)) or by visiting the
+releases by and downloading an `sbp_tools_haskell` archive.  This variant of
+the tools predate the Rust and Python versions, and also includes an `sbp2yaml`
+tool as well as a `sbp2prettyjson` tool.
+
+Finally, a Python version of the `sbp2json` tool exists, which is installable
+on any platform that support Python via pip, e.g.:
+
+```
+pip3 install sbp
+```
+
+The tool can then be invoked as follows:
+
+```
+python3 -m sbp2json <sbp.bin
+```
+
+On select platforms this Python tool is accelerated with native code and has
+performance on par with the Haskell and Rust variants.
+
 ## Building / installing
 ### Using Docker
 
@@ -156,9 +224,9 @@ HOWTO for instructions on updating these schemas.
 
 ## LICENSE
 
-Copyright © 2015 Swift Navigation
+Copyright © 2020 Swift Navigation
 
 Distributed under LGPLv3.0.
 
-[1]: https://travis-ci.org/swift-nav/libsbp.png
+[1]: https://travis-ci.org/swift-nav/libsbp.svg?branch=master
 [2]: https://travis-ci.org/swift-nav/libsbp

generator/sbpg/generator.py

@@ -174,6 +174,8 @@ def main():
       elif args.rust:
         parsed = [yaml.parse_spec(spec) for spec in file_index.values()]
         rs.render_mod(output_dir, parsed)
+        rs.render_sbp_cargo_toml(output_dir, args.release[0])
+        rs.render_sbp2json_cargo_toml(output_dir, args.release[0])
       elif args.test_c:
         test_c.render_check_suites(output_dir, all_specs)
         test_c.render_check_main(output_dir, all_specs)

generator/sbpg/targets/haskell.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
-# Copyright (C) 2015 Swift Navigation Inc.
-# Contact: Bhaskar Mookerji <mookerji@swiftnav.com>
+# Copyright (C) 2015-2020 Swift Navigation Inc.
+# Contact: Swift Navigation <dev@swiftnav.com>
 #
 # This source is subject to the license found in the file 'LICENSE' which must
 # be be distributed together with this source. All other rights reserved.