Skip to content

cli: --bench --json envelope with top-level engine field #419

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
danieljohnmorris merged 3 commits into from
May 18, 2026
Merged

cli: --bench --json envelope with top-level engine field #419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
95f80d9
cli: --bench honors --json with per-engine envelope
danieljohnmorris May 18, 2026
fc7a1b2
tests: cover bench --json engine field; pin eval_inline bench to --text
danieljohnmorris May 18, 2026
8609843
docs: ilo-engines.md notes the bench --json schema
danieljohnmorris May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions skills/ilo/ilo-engines.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,12 @@ Only the tree-walker runs **capturing** lambdas directly; VM, JIT, AOT detect ca
## Benchmarking

```
ilo file.ilo --bench main args... VM (default), reports ns/op
ilo file.ilo --jit --bench main args JIT
ilo file.ilo --run-tree --bench main Tree
ilo compile file.ilo -o ./prog && ./prog args AOT
ilo file.ilo --bench main args All engines, text
ilo file.ilo --bench main args --json All engines, JSON (one line per engine)
ilo compile file.ilo -o ./prog && ./prog args AOT (build then time)
```

JIT cold-start includes Cranelift compilation; for short programs that swamps the run time. Use `--bench` which loops the hot path.
`--bench` runs tree, vm, jit on the same input; JIT cold-start washes out in the hot loop. JSON envelope: `{"schemaVersion":1,"engine":"tree|vm|jit","variant":?,"result":...,"iterations":...,"totalMs":...,"perCallNs":...}`. VM emits two records (`variant: "fresh"` and `"reusable"`).

## AOT specifics

Expand Down
184 changes: 147 additions & 37 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2949,7 +2949,8 @@ fn dispatch_run(r: cli::RunArgs, mode: OutputMode, explicit_json: bool, no_hints
&[][..]
};
let run_args = parse_cli_args_typed(&program, func_name, raw);
run_bench(&program, func_name, &run_args);
let json = matches!(mode, OutputMode::Json);
run_bench(&program, func_name, &run_args, json);
0
} else if r.explain {
let filename = if is_file {
Expand Down Expand Up @@ -3865,7 +3866,54 @@ fn program_exit_code(val: &interpreter::Value) -> i32 {
}

#[allow(unused_variables, unused_mut)]
fn run_bench(program: &ast::Program, func_name: Option<&str>, args: &[interpreter::Value]) {
/// Emit one machine-readable JSON envelope for a single bench measurement.
///
/// One line per engine. Top-level `engine` field names which engine produced
/// the timing — `tree` (Rust interpreter), `vm` (register VM, with `variant`
/// distinguishing the fresh-compile vs reusable-VmState measurements), or
/// `jit` (Cranelift). `llvm` and `python` are emitted for completeness but
/// fall outside the four-engine `tree|vm|jit|aot` contract from adoption
/// brief 4; AOT isn't measured by `--bench` today (use `ilo build` then time
/// the binary). Text mode is unchanged — this only fires under `--json`.
fn emit_bench_json(
engine: &str,
variant: Option<&str>,
result: &str,
iterations: u32,
total_ms: f64,
per_call_ns: u128,
) {
// Escape the result string for JSON. Bench results are values rendered by
// `Display` so they can contain `"` and `\` (rare but possible — `Text`
// values pass through verbatim). Roll a minimal escaper here to avoid
// pulling serde just for one line.
let mut esc = String::with_capacity(result.len() + 2);
for c in result.chars() {
match c {
'"' => esc.push_str("\\\""),
'\\' => esc.push_str("\\\\"),
'\n' => esc.push_str("\\n"),
'\r' => esc.push_str("\\r"),
'\t' => esc.push_str("\\t"),
c if (c as u32) < 0x20 => esc.push_str(&format!("\\u{:04x}", c as u32)),
c => esc.push(c),
}
}
let variant_field = match variant {
Some(v) => format!(",\"variant\":\"{}\"", v),
None => String::new(),
};
println!(
"{{\"schemaVersion\":1,\"engine\":\"{engine}\"{variant_field},\"result\":\"{esc}\",\"iterations\":{iterations},\"totalMs\":{total_ms:.4},\"perCallNs\":{per_call_ns}}}"
);
}

fn run_bench(
program: &ast::Program,
func_name: Option<&str>,
args: &[interpreter::Value],
json: bool,
) {
use std::io::Write;
use std::process::Command;
use std::time::Instant;
Expand All @@ -3887,12 +3935,23 @@ fn run_bench(program: &ast::Program, func_name: Option<&str>, args: &[interprete
let interp_dur = start.elapsed();
let interp_ns = interp_dur.as_nanos() / iterations as u128;

println!("Rust interpreter");
println!(" result: {}", result);
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", interp_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", interp_ns);
println!();
if json {
emit_bench_json(
"tree",
None,
&result.to_string(),
iterations,
interp_dur.as_nanos() as f64 / 1e6,
interp_ns,
);
} else {
println!("Rust interpreter");
println!(" result: {}", result);
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", interp_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", interp_ns);
println!();
}

// -- Register VM benchmark --
let compiled = vm::compile(program).expect("compile error in benchmark");
Expand All @@ -3910,12 +3969,23 @@ fn run_bench(program: &ast::Program, func_name: Option<&str>, args: &[interprete
let vm_dur = start.elapsed();
let vm_ns = vm_dur.as_nanos() / iterations as u128;

println!("Register VM");
println!(" result: {}", vm_result);
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", vm_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", vm_ns);
println!();
if json {
emit_bench_json(
"vm",
Some("fresh"),
&vm_result.to_string(),
iterations,
vm_dur.as_nanos() as f64 / 1e6,
vm_ns,
);
} else {
println!("Register VM");
println!(" result: {}", vm_result);
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", vm_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", vm_ns);
println!();
}

// -- Register VM (reusable) benchmark --
let call_name = func_name.unwrap_or(
Expand All @@ -3939,15 +4009,26 @@ fn run_bench(program: &ast::Program, func_name: Option<&str>, args: &[interprete
let vm_reuse_dur = start.elapsed();
let vm_reuse_ns = vm_reuse_dur.as_nanos() / iterations as u128;

println!("Register VM (reusable)");
println!(" result: {}", vm_result);
println!(" iterations: {}", iterations);
println!(
" total: {:.2}ms",
vm_reuse_dur.as_nanos() as f64 / 1e6
);
println!(" per call: {}ns", vm_reuse_ns);
println!();
if json {
emit_bench_json(
"vm",
Some("reusable"),
&vm_result.to_string(),
iterations,
vm_reuse_dur.as_nanos() as f64 / 1e6,
vm_reuse_ns,
);
} else {
println!("Register VM (reusable)");
println!(" result: {}", vm_result);
println!(" iterations: {}", iterations);
println!(
" total: {:.2}ms",
vm_reuse_dur.as_nanos() as f64 / 1e6
);
println!(" per call: {}ns", vm_reuse_ns);
println!();
}

// -- JIT benchmarks --
// Extract function info for JIT
Expand Down Expand Up @@ -3997,12 +4078,23 @@ fn run_bench(program: &ast::Program, func_name: Option<&str>, args: &[interprete
// synthetic `<user_fn:N>` placeholder.
let jit_result =
vm::NanVal(jit_result_bits).to_value_with_program(&compiled.func_names);
println!("Cranelift JIT");
println!(" result: {}", jit_result);
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", jit_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", ns);
println!();
if json {
emit_bench_json(
"jit",
None,
&jit_result.to_string(),
iterations,
jit_dur.as_nanos() as f64 / 1e6,
ns,
);
} else {
println!("Cranelift JIT");
println!(" result: {}", jit_result);
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", jit_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", ns);
println!();
}
}
});
}
Expand All @@ -4029,21 +4121,39 @@ fn run_bench(program: &ast::Program, func_name: Option<&str>, args: &[interprete
let ns = jit_dur.as_nanos() / iterations as u128;
jit_llvm_ns = Some(ns);

println!("LLVM JIT");
if jit_result == (jit_result as i64) as f64 {
println!(" result: {}", jit_result as i64);
let result_str = if jit_result == (jit_result as i64) as f64 {
format!("{}", jit_result as i64)
} else {
println!(" result: {}", jit_result);
format!("{}", jit_result)
};
if json {
emit_bench_json(
"llvm",
None,
&result_str,
iterations,
jit_dur.as_nanos() as f64 / 1e6,
ns,
);
} else {
println!("LLVM JIT");
println!(" result: {}", result_str);
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", jit_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", ns);
println!();
}
println!(" iterations: {}", iterations);
println!(" total: {:.2}ms", jit_dur.as_nanos() as f64 / 1e6);
println!(" per call: {}ns", ns);
println!();
}
}
}

// -- Python transpiler benchmark (single invocation) --
// JSON mode skips Python entirely — it isn't one of the four engines
// (tree/vm/jit/aot) and the human-readable comparator only makes sense
// alongside the text-mode summary block below.
if json {
return;
}
let py_code = codegen::python::emit(program);
let call_func = func_name.unwrap_or("main").replace('-', "_");
let call_args: Vec<String> = args
Expand Down
14 changes: 7 additions & 7 deletions tests/eval_inline.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1395,7 +1395,7 @@ fn run_default_interpreter_error() {
fn bench_simple_function() {
// `f>n;42` with --bench covers run_bench (L448+), L230 (vec![]), all benchmark paths
let out = ilo()
.args(["f>n;42", "--bench", "f"])
.args(["f>n;42", "--bench", "f", "--text"])
.output()
.expect("failed to run ilo");
assert!(
Expand All @@ -1417,7 +1417,7 @@ fn bench_with_text_arg() {
// bench mode with a text arg → filter_map hits `_ => None` (L525), all_numeric=false,
// and the Python call_args builder hits the Text(s) branch (L638)
let out = ilo()
.args(["f x:t>t;x", "--bench", "f", "hello"])
.args(["f x:t>t;x", "--bench", "f", "hello", "--text"])
.output()
.expect("failed to run ilo");
assert!(
Expand All @@ -1437,7 +1437,7 @@ fn bench_with_text_arg() {
fn bench_with_bool_arg() {
// bench mode with a bool arg → Python call_args builder hits Bool(b) branch (L639)
let out = ilo()
.args(["f x:b>b;x", "--bench", "f", "true"])
.args(["f x:b>b;x", "--bench", "f", "true", "--text"])
.output()
.expect("failed to run ilo");
assert!(
Expand All @@ -1457,7 +1457,7 @@ fn bench_with_bool_arg() {
fn bench_with_list_arg() {
// bench mode with a list arg → Python call_args builder hits _ => "None" branch (L640)
let out = ilo()
.args(["f xs:L n>n;+xs.0 1", "--bench", "f", "[1,2,3]"])
.args(["f xs:L n>n;+xs.0 1", "--bench", "f", "[1,2,3]", "--text"])
.output()
.expect("failed to run ilo");
assert!(
Expand All @@ -1478,7 +1478,7 @@ fn bench_with_list_arg() {
fn bench_jit_float_result() {
// f x:n>n;/x 2 with arg 1 → JIT result = 0.5 (non-integer) → covers else branch
let out = ilo()
.args(["f x:n>n;/x 2", "--bench", "f", "1"])
.args(["f x:n>n;/x 2", "--bench", "f", "1", "--text"])
.output()
.expect("failed to run ilo");
assert!(
Expand Down Expand Up @@ -1507,7 +1507,7 @@ fn bench_jit_float_result() {
fn bench_jit_non_numeric_const() {
// f x:n>n;y="hi";x — NanVal JIT now handles text constants
let out = ilo()
.args(["f x:n>n;y=\"hi\";x", "--bench", "f", "5"])
.args(["f x:n>n;y=\"hi\";x", "--bench", "f", "5", "--text"])
.output()
.expect("failed to run ilo");
assert!(
Expand Down Expand Up @@ -1537,7 +1537,7 @@ fn bench_jit_move_different_regs() {
// compile_match_arms: result_reg = reg1, body compiles +x 1 to reg2
// → OP_MOVE 1,2 (a=1 != b=2) → arm64 L207-209 + Cranelift L167-170
let out = ilo()
.args(["f x:n>n;?x{_:+x 1}", "--bench", "f", "7"])
.args(["f x:n>n;?x{_:+x 1}", "--bench", "f", "7", "--text"])
.output()
.expect("failed to run ilo");
assert!(
Expand Down
Loading
Loading