Skip to content

Commit

Permalink
End-to-end cross-language roundtrip tests for our archetypes (#2601)
Browse files Browse the repository at this point in the history
With this PR, the CI will now compare the results of logging our
archetypes across all of our SDKs.

You can run it locally with:
```
./scripts/ci/run_e2e_roundtrip_tests.py --no-build
```

E.g.:

![image](https://github.com/rerun-io/rerun/assets/2910679/38f1541e-20ef-4ebd-8d55-ee6215fefd3c)


Requires #2597 

Fixes #2383 
Fixes #2384 
Fixes #2386 

---

### Checklist
* [x] I have read and agree to [Contributor
Guide](https://github.com/rerun-io/rerun/blob/main/CONTRIBUTING.md) and
the [Code of
Conduct](https://github.com/rerun-io/rerun/blob/main/CODE_OF_CONDUCT.md)
* [x] I've included a screenshot or gif (if applicable)
* [x] I have tested [demo.rerun.io](https://demo.rerun.io/pr/2601) (if
applicable)

- [PR Build Summary](https://build.rerun.io/pr/2601)
- [Docs
preview](https://rerun.io/preview/pr%3Acmc%2Fend_to_end_roundtrips_ci/docs)
- [Examples
preview](https://rerun.io/preview/pr%3Acmc%2Fend_to_end_roundtrips_ci/examples)
  • Loading branch information
teh-cmc committed Jul 6, 2023
1 parent b4efe9b commit 6cb2c68
Show file tree
Hide file tree
Showing 10 changed files with 249 additions and 5 deletions.
9 changes: 9 additions & 0 deletions .github/workflows/reusable_build_and_test_wheels.yml
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,15 @@ jobs:
shell: bash
run: RUST_LOG=debug scripts/run_python_e2e_test.py --no-build # rerun-sdk is already built and installed

- name: Run e2e roundtrip tests
if: needs.set-config.outputs.RUN_TESTS == 'true'
shell: bash
# --release so we can inherit from some of the artifacts that maturin has just built before
# --target x86_64-unknown-linux-gnu because otherwise cargo loses the target cache... even though this is the target anyhow...
# --no-build because rerun-sdk is already built and installed
# NOTE: run with --release so we can reuse some of the "Build Wheel"'s job artifacts, hopefully
run: RUST_LOG=debug scripts/ci/run_e2e_roundtrip_tests.py --release --target x86_64-unknown-linux-gnu --no-build

- name: Cache RRD dataset
if: needs.set-config.outputs.RUN_TESTS == 'true'
id: dataset
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,4 @@ screenshot*.png
web_demo

.nox/
out.rrd
9 changes: 9 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ members = [
"rerun_py",
"run_wasm",
"tests/rust/test_*",
"tests/rust/roundtrips/points2d",
]

[workspace.package]
Expand Down
2 changes: 1 addition & 1 deletion justfile
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ cpp-format:

### Python

py_folders := "examples rerun_py scripts docs/code-examples"
py_folders := "docs/code-examples examples rerun_py scripts tests"

# Set up a Pythonvirtual environment for development
py-dev-env:
Expand Down
108 changes: 108 additions & 0 deletions scripts/ci/run_e2e_roundtrip_tests.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#!/usr/bin/env python3

"""
Run our end-to-end cross-language roundtrip tests for all SDKs.
The list of archetypes is read directly from `crates/re_types/definitions/rerun/archetypes`.
If you create a new archetype definition without end-to-end tests, this will fail.
"""

from __future__ import annotations

import argparse
import os
import subprocess
import sys
import time
from os import listdir
from os.path import isfile, join

ARCHETYPES_PATH = "crates/re_types/definitions/rerun/archetypes"


def main() -> None:
parser = argparse.ArgumentParser(description="Run our end-to-end cross-language roundtrip tests for all SDK")
parser.add_argument("--no-build", action="store_true", help="Skip building rerun-sdk")
parser.add_argument("--release", action="store_true", help="Run cargo invocations with --release")
parser.add_argument("--target", type=str, default=None, help="Target used for cargo invocations")
parser.add_argument("--target-dir", type=str, default=None, help="Target directory used for cargo invocations")

args = parser.parse_args()

if args.no_build:
print("Skipping building rerun-sdk - assuming it is already built and up-to-date!")
else:
build_env = os.environ.copy()
if "RUST_LOG" in build_env:
del build_env["RUST_LOG"] # The user likely only meant it for the actual tests; not the setup

print("----------------------------------------------------------")
print("Building rerun-sdk…")
start_time = time.time()
subprocess.Popen(["just", "py-build"], env=build_env).wait()
elapsed = time.time() - start_time
print(f"rerun-sdk built in {elapsed:.1f} seconds")
print("")

files = [f for f in listdir(ARCHETYPES_PATH) if isfile(join(ARCHETYPES_PATH, f))]
archetypes = [filename for filename, extension in [os.path.splitext(file) for file in files] if extension == ".fbs"]

for arch in archetypes:
python_output_path = run_roundtrip_python(arch)
rust_output_path = run_roundtrip_rust(arch, args.release, args.target, args.target_dir)
run_comparison(python_output_path, rust_output_path)


def run_roundtrip_python(arch: str) -> str:
main_path = f"tests/python/roundtrips/{arch}/main.py"
output_path = f"tests/python/roundtrips/{arch}/out.rrd"

# sys.executable: the absolute path of the executable binary for the Python interpreter
python_executable = sys.executable
if python_executable is None:
python_executable = "python3"

cmd = [python_executable, main_path, "--save", output_path]
print(cmd)
roundtrip_process = subprocess.Popen(cmd)
returncode = roundtrip_process.wait(timeout=30)
assert returncode == 0, f"python roundtrip process exited with error code {returncode}"

return output_path


def run_roundtrip_rust(arch: str, release: bool, target: str | None, target_dir: str | None) -> str:
project_name = f"roundtrip_{arch}"
output_path = f"tests/rust/roundtrips/{arch}/out.rrd"

cmd = ["cargo", "r", "-p", project_name]

if target is not None:
cmd += ["--target", target]

if target_dir is not None:
cmd += ["--target-dir", target_dir]

if release:
cmd += ["--release"]

cmd += ["--", "--save", output_path]

print(cmd)
roundtrip_process = subprocess.Popen(cmd)
returncode = roundtrip_process.wait(timeout=12000)
assert returncode == 0, f"rust roundtrip process exited with error code {returncode}"

return output_path


def run_comparison(python_output_path: str, rust_output_path: str):
cmd = ["rerun", "compare", python_output_path, rust_output_path]
print(cmd)
roundtrip_process = subprocess.Popen(cmd)
returncode = roundtrip_process.wait(timeout=30)
assert returncode == 0, f"comparison process exited with error code {returncode}"


if __name__ == "__main__":
main()
54 changes: 54 additions & 0 deletions tests/python/roundtrips/points2d/main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
"""Logs a `Points2D` archetype for roundtrip checks."""

#!/usr/bin/env python3

from __future__ import annotations

import argparse

import numpy as np
import rerun as rr


def main() -> None:
points = np.array([1, 2, 3, 4], dtype=np.float32)
radii = np.array([0.42, 0.43], dtype=np.float32)
colors = np.array(
[
0xAA0000CC,
0x00BB00DD,
],
dtype=np.uint32,
)
labels = ["hello", "friend"]
draw_order = 300
class_ids = np.array([126, 127], dtype=np.uint64)
keypoint_ids = np.array([2, 3], dtype=np.uint64)
instance_keys = np.array([66, 666], dtype=np.uint64)

points2d = rr.Points2D(
points,
radii=radii,
colors=colors,
labels=labels,
draw_order=draw_order,
class_ids=class_ids,
keypoint_ids=keypoint_ids,
instance_keys=instance_keys,
)

parser = argparse.ArgumentParser(description="Logs rich data using the Rerun SDK.")
rr.script_add_args(parser)
args = parser.parse_args()

rr.script_setup(args, "roundtrip_points2d")

rr.log_any("points2d", points2d)
# Hack to establish 2d view bounds
rr.log_rect("rect", [0, 0, 4, 6])

rr.script_teardown(args)


if __name__ == "__main__":
main()
13 changes: 13 additions & 0 deletions tests/rust/roundtrips/points2d/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[package]
name = "roundtrip_points2d"
edition.workspace = true
license.workspace = true
publish = false
rust-version.workspace = true
version.workspace = true

[dependencies]
rerun = { path = "../../../../crates/rerun", features = ["native_viewer"] }

anyhow = "1.0"
clap = { version = "4.0", features = ["derive"] }
49 changes: 49 additions & 0 deletions tests/rust/roundtrips/points2d/src/main.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
//! Logs a `Points2D` archetype for roundtrip checks.

use rerun::{
components::Rect2D, experimental::archetypes::Points2D, external::re_log, MsgSender,
RecordingStream,
};

#[derive(Debug, clap::Parser)]
#[clap(author, version, about)]
struct Args {
#[command(flatten)]
rerun: rerun::clap::RerunArgs,
}

fn run(rec_stream: &RecordingStream, _args: &Args) -> anyhow::Result<()> {
MsgSender::from_archetype(
"points2d",
&Points2D::new([(1.0, 2.0), (3.0, 4.0)])
.with_radii([0.42, 0.43])
.with_colors([0xAA0000CC, 0x00BB00DD])
.with_labels(["hello", "friend"])
.with_draw_order(300.0)
.with_class_ids([126, 127])
.with_keypoint_ids([2, 3])
.with_instance_keys([66, 666]),
)?
.send(rec_stream)?;

// Hack to establish 2d view bounds
MsgSender::new("rect")
.with_component(&[Rect2D::from_xywh(0.0, 0.0, 4.0, 6.0)])?
.send(rec_stream)?;

Ok(())
}

fn main() -> anyhow::Result<()> {
re_log::setup_native_logging();

use clap::Parser as _;
let args = Args::parse();

let default_enabled = true;
args.rerun
.clone()
.run("roundtrip_points2d", default_enabled, move |rec_stream| {
run(&rec_stream, &args).unwrap();
})
}
8 changes: 4 additions & 4 deletions tests/rust/test_api/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
[package]
name = "test_api"
version = "0.8.0-alpha.0"
edition = "2021"
rust-version = "1.69"
license = "MIT OR Apache-2.0"
edition.workspace = true
license.workspace = true
publish = false
rust-version.workspace = true
version.workspace = true

[dependencies]
rerun = { path = "../../../crates/rerun", features = [
Expand Down

0 comments on commit 6cb2c68

Please sign in to comment.