From f0a1c46b99116c59047c127cb1dcd81e022a12e5 Mon Sep 17 00:00:00 2001 From: Sam Lewis Date: Mon, 27 Jun 2022 15:33:59 +1000 Subject: [PATCH 1/4] Add support for arrays > 32 in Rust [GV2-74] (#1174) The upcoming ed25519 signature message requires a fixed array of size 64 which currently breaks the Rust build because Serde does not currently natively support serializing or deserialzing arrays with sizes larger than 32. The `serde-big-array` crates allows us to work around this issue by using macros defined there for larger arrays. --- generator/sbpg/targets/resources/rust/sbp_cargo.toml | 6 +++++- generator/sbpg/targets/resources/rust/sbp_messages_mod.rs | 3 +++ .../sbpg/targets/resources/rust/sbp_messages_template.rs | 5 +++++ rust/sbp/Cargo.toml | 6 +++++- rust/sbp/src/messages/mod.rs | 3 +++ 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/generator/sbpg/targets/resources/rust/sbp_cargo.toml b/generator/sbpg/targets/resources/rust/sbp_cargo.toml index 770f97de62..0d9ded174b 100644 --- a/generator/sbpg/targets/resources/rust/sbp_cargo.toml +++ b/generator/sbpg/targets/resources/rust/sbp_cargo.toml @@ -20,7 +20,7 @@ readme = "../../README.md" [features] default = [] async = ["futures", "dencode/async"] -json = ["serde", "serde_json", "base64"] +json = ["serde", "serde_json", "serde-big-array", "base64"] link = ["slotmap"] [lib] @@ -48,6 +48,10 @@ optional = true version = "1.0" optional = true +[dependencies.serde-big-array] +version = "0.4.1" +optional = true + [dependencies.base64] version = "0.13" optional = true diff --git a/generator/sbpg/targets/resources/rust/sbp_messages_mod.rs b/generator/sbpg/targets/resources/rust/sbp_messages_mod.rs index d5e1d6bb6d..8b14ccced0 100644 --- a/generator/sbpg/targets/resources/rust/sbp_messages_mod.rs +++ b/generator/sbpg/targets/resources/rust/sbp_messages_mod.rs @@ -37,6 +37,9 @@ mod lib { pub use bytes::{Buf, BufMut}; + #[cfg(feature = "serde")] + pub use serde_big_array::BigArray; + macro_rules! get_bit_range { ($bitrange:expr, $source_ty:ty, $target_ty:ty, $msb:expr, $lsb:expr) => {{ let source_bit_len = std::mem::size_of::<$source_ty>() * 8; diff --git a/generator/sbpg/targets/resources/rust/sbp_messages_template.rs b/generator/sbpg/targets/resources/rust/sbp_messages_template.rs index d2fa4ff644..c1295b533d 100644 --- a/generator/sbpg/targets/resources/rust/sbp_messages_template.rs +++ b/generator/sbpg/targets/resources/rust/sbp_messages_template.rs @@ -53,7 +53,12 @@ pub struct (((m.msg_name))) { ((*- if f.desc *)) /// (((f.desc | commentify(indent=2) ))) ((*- endif *)) + ((*- if f.type_id == "array" and "size" in f.options and f.options["size"].value >= 32 *)) + #[cfg_attr(feature = "serde", serde(with="BigArray", rename(serialize = "(((f.identifier)))")))] + ((*- else *)) #[cfg_attr(feature = "serde", serde(rename(serialize = "(((f.identifier)))")))] + ((*- endif *)) + pub (((f.field_name))): (((f.type))), ((*- endfor *)) } diff --git a/rust/sbp/Cargo.toml b/rust/sbp/Cargo.toml index eb44ad8e82..194237594c 100644 --- a/rust/sbp/Cargo.toml +++ b/rust/sbp/Cargo.toml @@ -20,7 +20,7 @@ readme = "../../README.md" [features] default = [] async = ["futures", "dencode/async"] -json = ["serde", "serde_json", "base64"] +json = ["serde", "serde_json", "serde-big-array", "base64"] link = ["slotmap"] [lib] @@ -48,6 +48,10 @@ optional = true version = "1.0" optional = true +[dependencies.serde-big-array] +version = "0.4.1" +optional = true + [dependencies.base64] version = "0.13" optional = true diff --git a/rust/sbp/src/messages/mod.rs b/rust/sbp/src/messages/mod.rs index 3462ccb0a6..67e5786271 100644 --- a/rust/sbp/src/messages/mod.rs +++ b/rust/sbp/src/messages/mod.rs @@ -270,6 +270,9 @@ mod lib { pub use bytes::{Buf, BufMut}; + #[cfg(feature = "serde")] + pub use serde_big_array::BigArray; + macro_rules! get_bit_range { ($bitrange:expr, $source_ty:ty, $target_ty:ty, $msb:expr, $lsb:expr) => {{ let source_bit_len = std::mem::size_of::<$source_ty>() * 8; From c95d4ff6e3394f57dd70218a736eea04909fa82c Mon Sep 17 00:00:00 2001 From: Sam Lewis Date: Tue, 28 Jun 2022 09:34:31 +1000 Subject: [PATCH 2/4] java: Add support for u32 arrays [GV2-74] (#1175) java: Add support for u32 arrays The generated Java code was missing some code to allow for u32 arrays, which is needed for the upcoming ed25519 signature message. --- java/src/com/swiftnav/sbp/SBPMessage.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/java/src/com/swiftnav/sbp/SBPMessage.java b/java/src/com/swiftnav/sbp/SBPMessage.java index 805bdb5dae..0b3467d677 100644 --- a/java/src/com/swiftnav/sbp/SBPMessage.java +++ b/java/src/com/swiftnav/sbp/SBPMessage.java @@ -189,6 +189,16 @@ public int[] getArrayofU16(int n) { return ret; } + public long[] getArrayofU32() { + return getArrayofU32(buf.remaining() / 4); + } + + public long[] getArrayofU32(int n) { + long[] ret = new long[n]; + for (int i = 0; i < n; i++) ret[i] = getU32(); + return ret; + } + public float[] getArrayofFloat() { return getArrayofFloat(buf.remaining() / Float.BYTES); } @@ -341,6 +351,15 @@ public void putArrayofU16(int[] data, int n) { putArrayofU16(data); } + public void putArrayofU32(long[] data) { + for (long x : data) buf.putLong(x); + } + + public void putArrayofU32(long[] data, int n) { + assert (n == data.length); + putArrayofU32(data); + } + public void putArrayofDouble(double[] data) { for (double x : data) putDouble(x); } From 3a27cd10853d6498d8332ee2d5f6d3e1d77f3c7e Mon Sep 17 00:00:00 2001 From: Sam Lewis Date: Fri, 1 Jul 2022 14:00:49 +1000 Subject: [PATCH 3/4] Add example json2test instructions (#1177) Also installs some dependencies to the Dockerfile so that json2test can run in the libsbp docker image. --- Dockerfile | 2 +- HOWTO.md | 41 +++++++++++++++++++++++++++++++++++------ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index e07f3dc014..53953a3255 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,7 @@ ENV PATH=${SDKMAN_DIR}/candidates/gradle/current/bin:${PATH} RUN \ java --version \ && gradle --version \ - && pip3 install tox sphinx tox-run-command \ + && pip3 install tox sphinx tox-run-command construct pytest \ && curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal --no-modify-path \ && rustup component add rustfmt diff --git a/HOWTO.md b/HOWTO.md index 775d9838ad..5755b85779 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -63,10 +63,17 @@ process. This is likely to change in the future. client. We haven't quite decided on the details of this process. ## Generating missing tests - -Using `generator/missing.py` and `generator/json2test.py`, the yaml files -for test cases can be generated using either `missing.py` to listen via socket -or using `json2test.py` to translate json input files into yaml directly +There are some tools that can assist with generating YAML based tests, like the +ones already defined in the [`test`](spec/tests/yaml/swiftnav/sbp) directory. +These YAML files are used to generate tests in the various languages that libsbp +supports, to ensure that serializing and deserializing messages works as +intended + +### Existing Messages +For messages that are already being sent (eg: by Starling, or by a Piksi), the +`generator/missing.py` script can be used to connect to a socket and +automatically generate tests for any received messages that do not already have +tests. Usage for `missing`: @@ -74,7 +81,28 @@ Usage for `missing`: python missing.py --host [HOST] --port [PORT] ``` -* `missing.py` checks whether the message contains a test before writing one +### New Messages +The `json2test` script can be used to automatically generate tests for newly +defined messages. + +To use `json2test` a JSON file should be hand written with example contents of a +message. For example, to generate tests for the `MSG_HEARTBEAT` message (which +contains a single field named `flags`), you would generate a JSON file of the +form: + +```json +{ + "msg_type": 65535, + "flags": 12345, + "sender": 9876 +} +``` + +And then generate a test for using `json2test` with: + +```shell +PYTHONPATH="python/" python generator/json2test.py --input heartbeat.json --output spec/tests/yaml/swiftnav/sbp/system/test_MsgHeartbeat.yaml +``` Usage for `json2test` @@ -82,7 +110,8 @@ Usage for `json2test` python json2test --input [PATH_TO_JSON_IN] --output [PATH_TO_YAML_OUT] ``` -* can also provide message id with parameter `--msg-id [MESSAGE_ID]` +* The `msg_type` can also be provided through a CLI parameter, with `--msg-id + [MESSAGE_ID]` # Message Guidelines From c135afa21aa4610808e84dd0e976bbfd8db8b2f0 Mon Sep 17 00:00:00 2001 From: Sam Lewis Date: Fri, 1 Jul 2022 15:21:22 +1000 Subject: [PATCH 4/4] Add a tip to the HOWTO for new message groups (#1178) I was surprised that these files weren't auto-generated, but listing this as an extra step will hopefully avoid confusion for developers adding new message groups in the future. --- HOWTO.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/HOWTO.md b/HOWTO.md index 5755b85779..6fd3c6f398 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -40,7 +40,11 @@ process. This is likely to change in the future. 2. Increment `number_of_messages` in `python/tests/sbp/test_table.py` by the corresponding number of new messages. -3. Generate new clients and documentation by running `make +3. If adding a new "group" of messages (adding a new YAML file to + `spec/yaml/swiftnav/sbp`), add the new message group to + `python/sbp/table.py` and `javascript/sbp/msg.js`. + +4. Generate new clients and documentation by running `make all`. Verify that the generated code, which isn't too complicated, meets your expectations, as allowed messages are limited by the underlying language implementation. For example, you can't specify @@ -49,16 +53,16 @@ process. This is likely to change in the future. materialize a 0-length array C99 extension in the middle of the struct. GCC won't compile this. -4. (Optional) Add a [`test`](spec/tests/yaml/swiftnav/sbp) case and +5. (Optional) Add a [`test`](spec/tests/yaml/swiftnav/sbp) case and update the appropriate language libaries using `make gen`. If a test case is not added, increment `EXPECTED_MISSING_MESSAGES` in `python/tests/sbp/test_messages.py`. -5. Run `make test`. +6. Run `make test`. -6. Submit a pull request. +7. Submit a pull request. -7. If Swift's internal test tooling needs to be updated to use your +8. If Swift's internal test tooling needs to be updated to use your new message, deploy the updated Python client first, and then the C client. We haven't quite decided on the details of this process.