Add staging fixes to master

Dockerfile

@@ -99,7 +99,7 @@ ENV PATH=${SDKMAN_DIR}/candidates/gradle/current/bin:${PATH}
 RUN \
      java --version \
   && gradle --version \
-  && pip3 install tox sphinx tox-run-command \
+  && pip3 install tox sphinx tox-run-command construct pytest  \
   && curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal --no-modify-path \
   && rustup component add rustfmt
 

HOWTO.md

@@ -40,7 +40,11 @@ process. This is likely to change in the future.
 2. Increment `number_of_messages` in `python/tests/sbp/test_table.py`
    by the corresponding number of new messages.
 
-3. Generate new clients and documentation by running `make
+3. If adding a new "group" of messages (adding a new YAML file to
+   `spec/yaml/swiftnav/sbp`), add the new message group to 
+   `python/sbp/table.py` and `javascript/sbp/msg.js`. 
+
+4. Generate new clients and documentation by running `make
    all`. Verify that the generated code, which isn't too complicated,
    meets your expectations, as allowed messages are limited by the
    underlying language implementation. For example, you can't specify
@@ -49,40 +53,69 @@ process. This is likely to change in the future.
    materialize a 0-length array C99 extension in the middle of the
    struct. GCC won't compile this.
 
-4. (Optional) Add a [`test`](spec/tests/yaml/swiftnav/sbp) case and
+5. (Optional) Add a [`test`](spec/tests/yaml/swiftnav/sbp) case and
    update the appropriate language libaries using `make gen`.
    If a test case is not added, increment `EXPECTED_MISSING_MESSAGES`
    in `python/tests/sbp/test_messages.py`.
 
-5. Run `make test`.
+6. Run `make test`.
 
-6. Submit a pull request.
+7. Submit a pull request.
 
-7. If Swift's internal test tooling needs to be updated to use your
+8. If Swift's internal test tooling needs to be updated to use your
    new message, deploy the updated Python client first, and then the C
    client. We haven't quite decided on the details of this process.
 
 ##  Generating missing tests
-
-Using `generator/missing.py` and `generator/json2test.py`, the yaml files 
-for test cases can be generated using either `missing.py` to listen via socket
-or using `json2test.py` to translate json input files into yaml directly
+There are some tools that can assist with generating YAML based tests, like the
+ones already defined in the [`test`](spec/tests/yaml/swiftnav/sbp) directory.
+These YAML files are used to generate tests in the various languages that libsbp
+supports, to ensure that serializing and deserializing messages works as
+intended
+
+### Existing Messages
+For messages that are already being sent (eg: by Starling, or by a Piksi), the
+`generator/missing.py` script can be used to connect to a socket and
+automatically generate tests for any received messages that do not already have
+tests.
 
 Usage for `missing`:
 
 ```shell
 python missing.py --host [HOST] --port [PORT]
 ```
 
-* `missing.py` checks whether the message contains a test before writing one
+### New Messages
+The `json2test` script can be used to automatically generate tests for newly
+defined messages.
+
+To use `json2test` a JSON file should be hand written with example contents of a
+message. For example, to generate tests for the `MSG_HEARTBEAT` message (which
+contains a single field named `flags`), you would generate a JSON file of the
+form:
+
+```json
+{
+   "msg_type": 65535,
+   "flags": 12345,
+   "sender": 9876
+}
+```
+
+And then generate a test for using `json2test` with:
+
+```shell
+PYTHONPATH="python/" python generator/json2test.py --input heartbeat.json --output spec/tests/yaml/swiftnav/sbp/system/test_MsgHeartbeat.yaml
+```
 
 Usage for `json2test`
 
 ```shell
 python json2test --input [PATH_TO_JSON_IN] --output [PATH_TO_YAML_OUT]
 ```
 
-* can also provide message id with parameter `--msg-id [MESSAGE_ID]`
+* The `msg_type` can also be provided through a CLI parameter, with `--msg-id
+  [MESSAGE_ID]`
 
 # Message Guidelines
 

generator/sbpg/targets/resources/rust/sbp_cargo.toml

@@ -20,7 +20,7 @@ readme = "../../README.md"
 [features]
 default = []
 async = ["futures", "dencode/async"]
-json = ["serde", "serde_json", "base64"]
+json = ["serde", "serde_json", "serde-big-array", "base64"]
 link = ["slotmap"]
 
 [lib]
@@ -48,6 +48,10 @@ optional = true
 version = "1.0"
 optional = true
 
+[dependencies.serde-big-array]
+version = "0.4.1"
+optional = true
+
 [dependencies.base64]
 version = "0.13"
 optional = true

generator/sbpg/targets/resources/rust/sbp_messages_mod.rs

@@ -37,6 +37,9 @@ mod lib {
 
     pub use bytes::{Buf, BufMut};
 
+    #[cfg(feature = "serde")]
+    pub use serde_big_array::BigArray;
+
     macro_rules! get_bit_range {
         ($bitrange:expr, $source_ty:ty, $target_ty:ty, $msb:expr, $lsb:expr) => {{
             let source_bit_len = std::mem::size_of::<$source_ty>() * 8;

generator/sbpg/targets/resources/rust/sbp_messages_template.rs

@@ -53,7 +53,12 @@ pub struct (((m.msg_name))) {
     ((*- if f.desc *))
     /// (((f.desc | commentify(indent=2) )))
     ((*- endif *))
+    ((*- if f.type_id == "array" and "size" in f.options and f.options["size"].value >= 32 *))
+    #[cfg_attr(feature = "serde", serde(with="BigArray", rename(serialize = "(((f.identifier)))")))]
+    ((*- else *))
     #[cfg_attr(feature = "serde", serde(rename(serialize = "(((f.identifier)))")))]
+    ((*- endif *))
+
     pub (((f.field_name))): (((f.type))),
     ((*- endfor *))
 }

java/src/com/swiftnav/sbp/SBPMessage.java

@@ -189,6 +189,16 @@ public int[] getArrayofU16(int n) {
             return ret;
         }
 
+        public long[] getArrayofU32() {
+            return getArrayofU32(buf.remaining() / 4);
+        }
+
+        public long[] getArrayofU32(int n) {
+            long[] ret = new long[n];
+            for (int i = 0; i < n; i++) ret[i] = getU32();
+            return ret;
+        }
+
         public float[] getArrayofFloat() {
             return getArrayofFloat(buf.remaining() / Float.BYTES);
         }
@@ -341,6 +351,15 @@ public void putArrayofU16(int[] data, int n) {
             putArrayofU16(data);
         }
 
+        public void putArrayofU32(long[] data) {
+            for (long x : data) buf.putLong(x);
+        }
+
+        public void putArrayofU32(long[] data, int n) {
+            assert (n == data.length);
+            putArrayofU32(data);
+        }
+
         public void putArrayofDouble(double[] data) {
             for (double x : data) putDouble(x);
         }

rust/sbp/Cargo.toml

@@ -20,7 +20,7 @@ readme = "../../README.md"
 [features]
 default = []
 async = ["futures", "dencode/async"]
-json = ["serde", "serde_json", "base64"]
+json = ["serde", "serde_json", "serde-big-array", "base64"]
 link = ["slotmap"]
 
 [lib]
@@ -48,6 +48,10 @@ optional = true
 version = "1.0"
 optional = true
 
+[dependencies.serde-big-array]
+version = "0.4.1"
+optional = true
+
 [dependencies.base64]
 version = "0.13"
 optional = true

rust/sbp/src/messages/mod.rs

@@ -270,6 +270,9 @@ mod lib {
 
     pub use bytes::{Buf, BufMut};
 
+    #[cfg(feature = "serde")]
+    pub use serde_big_array::BigArray;
+
     macro_rules! get_bit_range {
         ($bitrange:expr, $source_ty:ty, $target_ty:ty, $msb:expr, $lsb:expr) => {{
             let source_bit_len = std::mem::size_of::<$source_ty>() * 8;